feat: increase message and batch sizes (#108)

* fix(flags): Ensure feature properties exist on feature flag called events

* fix

* make flake8 happy, will this bork?

.github/workflows/ci.yml

@@ -33,6 +33,10 @@ jobs:
             - name: Check formatting with black
               run: |
                   black --check .
+            
+            - name: Lint with flake8
+              run: |
+                  flake8 posthog --ignore E501
 
             - name: Check import order with isort
               run: |
@@ -58,4 +62,4 @@ jobs:
 
             - name: Run posthog tests
               run: |
-                  python setup.py test
+                  pytest --verbose --timeout=30

CHANGELOG.md

@@ -1,3 +1,46 @@
+## 3.1.0 - 2023-12-04
+
+1. Increase maximum event size and batch size
+
+## 3.0.2 - 2023-08-17
+
+1. Returns the current flag property with $feature_flag_called events, to make it easier to use in experiments
+
+## 3.0.1 - 2023-04-21
+
+1. Restore how feature flags work when the client library is disabled: All requests return `None` and no events are sent when the client is disabled.
+2. Add a `feature_flag_definitions()` debug option, which returns currently loaded feature flag definitions. You can use this to more cleverly decide when to request local evaluation of feature flags.
+## 3.0.0 - 2023-04-14
+
+Breaking change:
+
+All events by default now send the `$geoip_disable` property to disable geoip lookup in app. This is because usually we don't
+want to update person properties to take the server's location.
+
+The same now happens for feature flag requests, where we discard the IP address of the server for matching on geoip properties like city, country, continent.
+
+To restore previous behaviour, you can set the default to False like so:
+
+```python
+posthog.disable_geoip = False
+
+# // and if using client instantiation:
+posthog = Posthog('api_key', disable_geoip=False)
+
+```
+
+## 2.5.0 - 2023-04-10
+
+1. Add option for instantiating separate client object
+
+## 2.4.2 - 2023-03-30
+
+1. Update backoff dependency for posthoganalytics package to be the same as posthog package
+
+## 2.4.1 - 2023-03-17
+
+1. Removes accidental print call left in for decide response
+
 ## 2.4.0 - 2023-03-14
 
 1. Support evaluating all cohorts in feature flags for local evaluation

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020 PostHog (part of Hiberly Inc)
+Copyright (c) 2023 PostHog (part of Hiberly Inc)
 
 Copyright (c) 2013 Segment Inc. friends@segment.com
 

posthog/__init__.py

@@ -1,4 +1,5 @@
-from typing import Callable, Dict, Optional
+import datetime  # noqa: F401
+from typing import Callable, Dict, Optional  # noqa: F401
 
 from posthog.client import Client
 from posthog.version import VERSION
@@ -16,6 +17,7 @@ disabled = False  # type: bool
 personal_api_key = None  # type: str
 project_api_key = None  # type: str
 poll_interval = 30  # type: int
+disable_geoip = True  # type: bool
 
 default_client = None
 
@@ -29,6 +31,7 @@ def capture(
     uuid=None,  # type: Optional[str]
     groups=None,  # type: Optional[Dict]
     send_feature_flags=False,
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -61,6 +64,7 @@ def capture(
         uuid=uuid,
         groups=groups,
         send_feature_flags=send_feature_flags,
+        disable_geoip=disable_geoip,
     )
 
 
@@ -70,6 +74,7 @@ def identify(
     context=None,  # type: Optional[Dict]
     timestamp=None,  # type: Optional[datetime.datetime]
     uuid=None,  # type: Optional[str]
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -94,15 +99,17 @@ def identify(
         context=context,
         timestamp=timestamp,
         uuid=uuid,
+        disable_geoip=disable_geoip,
     )
 
 
 def set(
-    distinct_id,  # type: str,
+    distinct_id,  # type: str
     properties=None,  # type: Optional[Dict]
     context=None,  # type: Optional[Dict]
     timestamp=None,  # type: Optional[datetime.datetime]
     uuid=None,  # type: Optional[str]
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -127,15 +134,17 @@ def set(
         context=context,
         timestamp=timestamp,
         uuid=uuid,
+        disable_geoip=disable_geoip,
     )
 
 
 def set_once(
-    distinct_id,  # type: str,
+    distinct_id,  # type: str
     properties=None,  # type: Optional[Dict]
     context=None,  # type: Optional[Dict]
     timestamp=None,  # type: Optional[datetime.datetime]
     uuid=None,  # type: Optional[str]
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -160,6 +169,7 @@ def set_once(
         context=context,
         timestamp=timestamp,
         uuid=uuid,
+        disable_geoip=disable_geoip,
     )
 
 
@@ -170,6 +180,7 @@ def group_identify(
     context=None,  # type: Optional[Dict]
     timestamp=None,  # type: Optional[datetime.datetime]
     uuid=None,  # type: Optional[str]
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -195,15 +206,17 @@ def group_identify(
         context=context,
         timestamp=timestamp,
         uuid=uuid,
+        disable_geoip=disable_geoip,
     )
 
 
 def alias(
-    previous_id,  # type: str,
-    distinct_id,  # type: str,
+    previous_id,  # type: str
+    distinct_id,  # type: str
     context=None,  # type: Optional[Dict]
     timestamp=None,  # type: Optional[datetime.datetime]
     uuid=None,  # type: Optional[str]
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> None
     """
@@ -229,17 +242,19 @@ def alias(
         context=context,
         timestamp=timestamp,
         uuid=uuid,
+        disable_geoip=disable_geoip,
     )
 
 
 def feature_enabled(
-    key,  # type: str,
-    distinct_id,  # type: str,
+    key,  # type: str
+    distinct_id,  # type: str
     groups={},  # type: dict
     person_properties={},  # type: dict
     group_properties={},  # type: dict
     only_evaluate_locally=False,  # type: bool
     send_feature_flag_events=True,  # type: bool
+    disable_geoip=None,  # type: Optional[bool]
 ):
     # type: (...) -> bool
     """
@@ -264,17 +279,19 @@ def feature_enabled(
         group_properties=group_properties,
         only_evaluate_locally=only_evaluate_locally,
         send_feature_flag_events=send_feature_flag_events,
+        disable_geoip=disable_geoip,
     )
 
 
 def get_feature_flag(
-    key,  # type: str,
-    distinct_id,  # type: str,
+    key,  # type: str
+    distinct_id,  # type: str
     groups={},  # type: dict
     person_properties={},  # type: dict
     group_properties={},  # type: dict
     only_evaluate_locally=False,  # type: bool
     send_feature_flag_events=True,  # type: bool
+    disable_geoip=None,  # type: Optional[bool]
 ):
     """
     Get feature flag variant for users. Used with experiments.
@@ -307,15 +324,17 @@ def get_feature_flag(
         group_properties=group_properties,
         only_evaluate_locally=only_evaluate_locally,
         send_feature_flag_events=send_feature_flag_events,
+        disable_geoip=disable_geoip,
     )
 
 
 def get_all_flags(
-    distinct_id,  # type: str,
+    distinct_id,  # type: str
     groups={},  # type: dict
     person_properties={},  # type: dict
     group_properties={},  # type: dict
     only_evaluate_locally=False,  # type: bool
+    disable_geoip=None,  # type: Optional[bool]
 ):
     """
     Get all flags for a given user.
@@ -333,6 +352,7 @@ def get_all_flags(
         person_properties=person_properties,
         group_properties=group_properties,
         only_evaluate_locally=only_evaluate_locally,
+        disable_geoip=disable_geoip,
     )
 
 
@@ -345,6 +365,7 @@ def get_feature_flag_payload(
     group_properties={},
     only_evaluate_locally=False,
     send_feature_flag_events=True,
+    disable_geoip=None,  # type: Optional[bool]
 ):
     return _proxy(
         "get_feature_flag_payload",
@@ -356,6 +377,7 @@ def get_feature_flag_payload(
         group_properties=group_properties,
         only_evaluate_locally=only_evaluate_locally,
         send_feature_flag_events=send_feature_flag_events,
+        disable_geoip=disable_geoip,
     )
 
 
@@ -365,6 +387,7 @@ def get_all_flags_and_payloads(
     person_properties={},
     group_properties={},
     only_evaluate_locally=False,
+    disable_geoip=None,  # type: Optional[bool]
 ):
     return _proxy(
         "get_all_flags_and_payloads",
@@ -373,9 +396,15 @@ def get_all_flags_and_payloads(
         person_properties=person_properties,
         group_properties=group_properties,
         only_evaluate_locally=only_evaluate_locally,
+        disable_geoip=disable_geoip,
     )
 
 
+def feature_flag_definitions():
+    """Returns loaded feature flags, if any. Helpful for debugging what flag information you have loaded."""
+    return _proxy("feature_flag_definitions")
+
+
 def page(*args, **kwargs):
     """Send a page call."""
     _proxy("page", *args, **kwargs)
@@ -405,8 +434,6 @@ def shutdown():
 def _proxy(method, *args, **kwargs):
     """Create an analytics client if one doesn't exist and send to it."""
     global default_client
-    if disabled:
-        return None
     if not default_client:
         default_client = Client(
             api_key,
@@ -418,7 +445,17 @@ def _proxy(method, *args, **kwargs):
             personal_api_key=personal_api_key,
             project_api_key=project_api_key,
             poll_interval=poll_interval,
+            disabled=disabled,
+            disable_geoip=disable_geoip,
         )
 
+    # always set incase user changes it
+    default_client.disabled = disabled
+    default_client.debug = debug
+
     fn = getattr(default_client, method)
     return fn(*args, **kwargs)
+
+
+class Posthog(Client):
+    pass

posthog/client.py

@@ -47,6 +47,8 @@ class Client(object):
         poll_interval=30,
         personal_api_key=None,
         project_api_key=None,
+        disabled=False,
+        disable_geoip=True,
     ):
         self.queue = queue.Queue(max_queue_size)
 
@@ -69,6 +71,8 @@ class Client(object):
         self.poll_interval = poll_interval
         self.poller = None
         self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set)
+        self.disabled = disabled
+        self.disable_geoip = disable_geoip
 
         # personal_api_key: This should be a generated Personal API Key, private
         self.personal_api_key = personal_api_key
@@ -110,7 +114,7 @@ class Client(object):
                 if send:
                     consumer.start()
 
-    def identify(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None):
+    def identify(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None, disable_geoip=None):
         properties = properties or {}
         context = context or {}
         require("distinct_id", distinct_id, ID_TYPES)
@@ -125,25 +129,36 @@ class Client(object):
             "uuid": uuid,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def get_feature_variants(self, distinct_id, groups=None, person_properties=None, group_properties=None):
-        resp_data = self.get_decide(distinct_id, groups, person_properties, group_properties)
+    def get_feature_variants(
+        self, distinct_id, groups=None, person_properties=None, group_properties=None, disable_geoip=None
+    ):
+        resp_data = self.get_decide(distinct_id, groups, person_properties, group_properties, disable_geoip)
         return resp_data["featureFlags"]
 
-    def _get_active_feature_variants(self, distinct_id, groups=None, person_properties=None, group_properties=None):
-        feature_variants = self.get_feature_variants(distinct_id, groups, person_properties, group_properties)
+    def _get_active_feature_variants(
+        self, distinct_id, groups=None, person_properties=None, group_properties=None, disable_geoip=None
+    ):
+        feature_variants = self.get_feature_variants(
+            distinct_id, groups, person_properties, group_properties, disable_geoip
+        )
         return {
             k: v for (k, v) in feature_variants.items() if v is not False
         }  # explicitly test for false to account for values that may seem falsy (ex: 0)
 
-    def get_feature_payloads(self, distinct_id, groups=None, person_properties=None, group_properties=None):
-        resp_data = self.get_decide(distinct_id, groups, person_properties, group_properties)
+    def get_feature_payloads(
+        self, distinct_id, groups=None, person_properties=None, group_properties=None, disable_geoip=None
+    ):
+        resp_data = self.get_decide(distinct_id, groups, person_properties, group_properties, disable_geoip)
         return resp_data["featureFlagPayloads"]
 
-    def get_decide(self, distinct_id, groups=None, person_properties=None, group_properties=None):
+    def get_decide(self, distinct_id, groups=None, person_properties=None, group_properties=None, disable_geoip=None):
         require("distinct_id", distinct_id, ID_TYPES)
 
+        if disable_geoip is None:
+            disable_geoip = self.disable_geoip
+
         if groups:
             require("groups", groups, dict)
         else:
@@ -154,10 +169,10 @@ class Client(object):
             "groups": groups,
             "person_properties": person_properties,
             "group_properties": group_properties,
+            "disable_geoip": disable_geoip,
         }
         resp_data = decide(self.api_key, self.host, timeout=10, **request_data)
 
-        print(resp_data)
         return resp_data
 
     def capture(
@@ -170,6 +185,7 @@ class Client(object):
         uuid=None,
         groups=None,
         send_feature_flags=False,
+        disable_geoip=None,
     ):
         properties = properties or {}
         context = context or {}
@@ -192,7 +208,7 @@ class Client(object):
 
         if send_feature_flags:
             try:
-                feature_variants = self._get_active_feature_variants(distinct_id, groups)
+                feature_variants = self._get_active_feature_variants(distinct_id, groups, disable_geoip=disable_geoip)
             except Exception as e:
                 self.log.exception(f"[FEATURE FLAGS] Unable to get feature variants: {e}")
             else:
@@ -200,9 +216,9 @@ class Client(object):
                     msg["properties"]["$feature/{}".format(feature)] = variant
                 msg["properties"]["$active_feature_flags"] = list(feature_variants.keys())
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def set(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None):
+    def set(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None, disable_geoip=None):
         properties = properties or {}
         context = context or {}
         require("distinct_id", distinct_id, ID_TYPES)
@@ -217,9 +233,9 @@ class Client(object):
             "uuid": uuid,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def set_once(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None):
+    def set_once(self, distinct_id=None, properties=None, context=None, timestamp=None, uuid=None, disable_geoip=None):
         properties = properties or {}
         context = context or {}
         require("distinct_id", distinct_id, ID_TYPES)
@@ -234,9 +250,18 @@ class Client(object):
             "uuid": uuid,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def group_identify(self, group_type=None, group_key=None, properties=None, context=None, timestamp=None, uuid=None):
+    def group_identify(
+        self,
+        group_type=None,
+        group_key=None,
+        properties=None,
+        context=None,
+        timestamp=None,
+        uuid=None,
+        disable_geoip=None,
+    ):
         properties = properties or {}
         context = context or {}
         require("group_type", group_type, ID_TYPES)
@@ -256,9 +281,9 @@ class Client(object):
             "uuid": uuid,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def alias(self, previous_id=None, distinct_id=None, context=None, timestamp=None, uuid=None):
+    def alias(self, previous_id=None, distinct_id=None, context=None, timestamp=None, uuid=None, disable_geoip=None):
         context = context or {}
 
         require("previous_id", previous_id, ID_TYPES)
@@ -275,9 +300,11 @@ class Client(object):
             "distinct_id": previous_id,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def page(self, distinct_id=None, url=None, properties=None, context=None, timestamp=None, uuid=None):
+    def page(
+        self, distinct_id=None, url=None, properties=None, context=None, timestamp=None, uuid=None, disable_geoip=None
+    ):
         properties = properties or {}
         context = context or {}
 
@@ -296,10 +323,14 @@ class Client(object):
             "uuid": uuid,
         }
 
-        return self._enqueue(msg)
+        return self._enqueue(msg, disable_geoip)
 
-    def _enqueue(self, msg):
+    def _enqueue(self, msg, disable_geoip):
         """Push a new `msg` onto the queue, return `(success, msg)`"""
+
+        if self.disabled:
+            return False, "disabled"
+
         timestamp = msg["timestamp"]
         if timestamp is None:
             timestamp = datetime.utcnow().replace(tzinfo=tzutc())
@@ -322,6 +353,12 @@ class Client(object):
         msg["properties"]["$lib"] = "posthog-python"
         msg["properties"]["$lib_version"] = VERSION
 
+        if disable_geoip is None:
+            disable_geoip = self.disable_geoip
+
+        if disable_geoip:
+            msg["properties"]["$geoip_disable"] = True
+
         msg["distinct_id"] = stringify_id(msg.get("distinct_id", None))
 
         msg = clean(msg)
@@ -392,7 +429,7 @@ class Client(object):
         except APIError as e:
             if e.status == 401:
                 self.log.error(
-                    f"[FEATURE FLAGS] Error loading feature flags: To use feature flags, please set a valid personal_api_key. More information: https://posthog.com/docs/api/overview"
+                    "[FEATURE FLAGS] Error loading feature flags: To use feature flags, please set a valid personal_api_key. More information: https://posthog.com/docs/api/overview"
                 )
                 if self.debug:
                     raise APIError(
@@ -465,6 +502,7 @@ class Client(object):
         group_properties={},
         only_evaluate_locally=False,
         send_feature_flag_events=True,
+        disable_geoip=None,
     ):
         response = self.get_feature_flag(
             key,
@@ -474,6 +512,7 @@ class Client(object):
             group_properties=group_properties,
             only_evaluate_locally=only_evaluate_locally,
             send_feature_flag_events=send_feature_flag_events,
+            disable_geoip=disable_geoip,
         )
 
         if response is None:
@@ -490,12 +529,16 @@ class Client(object):
         group_properties={},
         only_evaluate_locally=False,
         send_feature_flag_events=True,
+        disable_geoip=None,
     ):
         require("key", key, string_types)
         require("distinct_id", distinct_id, ID_TYPES)
         require("groups", groups, dict)
 
-        if self.feature_flags == None and self.personal_api_key:
+        if self.disabled:
+            return None
+
+        if self.feature_flags is None and self.personal_api_key:
             self.load_feature_flags()
         response = None
 
@@ -523,7 +566,11 @@ class Client(object):
         if not flag_was_locally_evaluated and not only_evaluate_locally:
             try:
                 feature_flags = self.get_feature_variants(
-                    distinct_id, groups=groups, person_properties=person_properties, group_properties=group_properties
+                    distinct_id,
+                    groups=groups,
+                    person_properties=person_properties,
+                    group_properties=group_properties,
+                    disable_geoip=disable_geoip,
                 )
                 response = feature_flags.get(key)
                 if response is None:
@@ -535,7 +582,7 @@ class Client(object):
         feature_flag_reported_key = f"{key}_{str(response)}"
         if (
             feature_flag_reported_key not in self.distinct_ids_feature_flags_reported[distinct_id]
-            and send_feature_flag_events
+            and send_feature_flag_events  # noqa: W503
         ):
             self.capture(
                 distinct_id,
@@ -544,8 +591,10 @@ class Client(object):
                     "$feature_flag": key,
                     "$feature_flag_response": response,
                     "locally_evaluated": flag_was_locally_evaluated,
+                    f"$feature/{key}": response,
                 },
                 groups=groups,
+                disable_geoip=disable_geoip,
             )
             self.distinct_ids_feature_flags_reported[distinct_id].add(feature_flag_reported_key)
         return response
@@ -561,7 +610,11 @@ class Client(object):
         group_properties={},
         only_evaluate_locally=False,
         send_feature_flag_events=True,
+        disable_geoip=None,
     ):
+        if self.disabled:
+            return None
+
         if match_value is None:
             match_value = self.get_feature_flag(
                 key,
@@ -571,6 +624,7 @@ class Client(object):
                 group_properties=group_properties,
                 send_feature_flag_events=send_feature_flag_events,
                 only_evaluate_locally=True,
+                disable_geoip=disable_geoip,
             )
 
         response = None
@@ -579,7 +633,9 @@ class Client(object):
             response = self._compute_payload_locally(key, match_value)
 
         if response is None and not only_evaluate_locally:
-            decide_payloads = self.get_feature_payloads(distinct_id, groups, person_properties, group_properties)
+            decide_payloads = self.get_feature_payloads(
+                distinct_id, groups, person_properties, group_properties, disable_geoip
+            )
             response = decide_payloads.get(str(key).lower(), None)
 
         return response
@@ -597,7 +653,14 @@ class Client(object):
         return payload
 
     def get_all_flags(
-        self, distinct_id, *, groups={}, person_properties={}, group_properties={}, only_evaluate_locally=False
+        self,
+        distinct_id,
+        *,
+        groups={},
+        person_properties={},
+        group_properties={},
+        only_evaluate_locally=False,
+        disable_geoip=None,
     ):
         flags = self.get_all_flags_and_payloads(
             distinct_id,
@@ -605,12 +668,23 @@ class Client(object):
             person_properties=person_properties,
             group_properties=group_properties,
             only_evaluate_locally=only_evaluate_locally,
+            disable_geoip=disable_geoip,
         )
         return flags["featureFlags"]
 
     def get_all_flags_and_payloads(
-        self, distinct_id, *, groups={}, person_properties={}, group_properties={}, only_evaluate_locally=False
+        self,
+        distinct_id,
+        *,
+        groups={},
+        person_properties={},
+        group_properties={},
+        only_evaluate_locally=False,
+        disable_geoip=None,
     ):
+        if self.disabled:
+            return {"featureFlags": None, "featureFlagPayloads": None}
+
         flags, payloads, fallback_to_decide = self._get_all_flags_and_payloads_locally(
             distinct_id, groups=groups, person_properties=person_properties, group_properties=group_properties
         )
@@ -619,7 +693,11 @@ class Client(object):
         if fallback_to_decide and not only_evaluate_locally:
             try:
                 flags_and_payloads = self.get_decide(
-                    distinct_id, groups=groups, person_properties=person_properties, group_properties=group_properties
+                    distinct_id,
+                    groups=groups,
+                    person_properties=person_properties,
+                    group_properties=group_properties,
+                    disable_geoip=disable_geoip,
                 )
                 response = flags_and_payloads
             except Exception as e:
@@ -631,7 +709,7 @@ class Client(object):
         require("distinct_id", distinct_id, ID_TYPES)
         require("groups", groups, dict)
 
-        if self.feature_flags == None and self.personal_api_key:
+        if self.feature_flags is None and self.personal_api_key:
             self.load_feature_flags()
 
         flags = {}
@@ -651,7 +729,7 @@ class Client(object):
                     matched_payload = self._compute_payload_locally(flag["key"], flags[flag["key"]])
                     if matched_payload:
                         payloads[flag["key"]] = matched_payload
-                except InconclusiveMatchError as e:
+                except InconclusiveMatchError:
                     # No need to log this, since it's just telling us to fall back to `/decide`
                     fallback_to_decide = True
                 except Exception as e:
@@ -662,6 +740,9 @@ class Client(object):
 
         return flags, payloads, fallback_to_decide
 
+    def feature_flag_definitions(self):
+        return self.feature_flags
+
 
 def require(name, field, data_type):
     """Require that the named `field` has the right `data_type`"""

posthog/consumer.py

@@ -12,11 +12,12 @@ try:
 except ImportError:
     from Queue import Empty
 
-MAX_MSG_SIZE = 32 << 10
 
-# Our servers only accept batches less than 500KB. Here limit is set slightly
-# lower to leave space for extra data that will be added later, eg. "sentAt".
-BATCH_SIZE_LIMIT = 475000
+MAX_MSG_SIZE = 900 * 1024  # 900KiB per event
+
+# The maximum request body size is currently 20MiB, let's be conservative
+# in case we want to lower it in the future.
+BATCH_SIZE_LIMIT = 5 * 1024 * 1024
 
 
 class Consumer(Thread):
@@ -104,7 +105,7 @@ class Consumer(Thread):
                 item = queue.get(block=True, timeout=self.flush_interval - elapsed)
                 item_size = len(json.dumps(item, cls=DatetimeSerializer).encode())
                 if item_size > MAX_MSG_SIZE:
-                    self.log.error("Item exceeds 32kb limit, dropping. (%s)", str(item))
+                    self.log.error("Item exceeds 900kib limit, dropping. (%s)", str(item))
                     continue
                 items.append(item)
                 total_size += item_size

posthog/feature_flags.py

@@ -143,16 +143,16 @@ def match_property(property, property_values) -> bool:
         return is_valid_regex(str(value)) and re.compile(str(value)).search(str(override_value)) is None
 
     if operator == "gt":
-        return type(override_value) == type(value) and override_value > value
+        return type(override_value) is type(value) and override_value > value
 
     if operator == "gte":
-        return type(override_value) == type(value) and override_value >= value
+        return type(override_value) is type(value) and override_value >= value
 
     if operator == "lt":
-        return type(override_value) == type(value) and override_value < value
+        return type(override_value) is type(value) and override_value < value
 
     if operator == "lte":
-        return type(override_value) == type(value) and override_value <= value
+        return type(override_value) is type(value) and override_value <= value
 
     if operator in ["is_date_before", "is_date_after"]:
         try:

posthog/sentry/django.py

@@ -11,7 +11,7 @@ def get_distinct_id(request):
         return None
     try:
         return GET_DISTINCT_ID(request)
-    except:
+    except:  # noqa: E722
         return None
 
 

posthog/sentry/posthog_integration.py

@@ -1,5 +1,4 @@
 from sentry_sdk._types import MYPY
-from sentry_sdk.client import Client
 from sentry_sdk.hub import Hub
 from sentry_sdk.integrations import Integration
 from sentry_sdk.scope import add_global_event_processor
@@ -10,9 +9,9 @@ from posthog.request import DEFAULT_HOST
 from posthog.sentry import POSTHOG_ID_TAG
 
 if MYPY:
-    from typing import Any, Dict, Optional
+    from typing import Optional  # noqa: F401
 
-    from sentry_sdk._types import Event, Hint
+    from sentry_sdk._types import Event, Hint  # noqa: F401
 
 
 class PostHogIntegration(Integration):

posthog/test/test_client.py

@@ -1,7 +1,6 @@
 import time
 import unittest
-from datetime import date, datetime
-from unittest.mock import MagicMock
+from datetime import datetime
 from uuid import uuid4
 
 import mock
@@ -28,7 +27,7 @@ class TestClient(unittest.TestCase):
 
     def set_fail(self, e, batch):
         """Mark the failure handler"""
-        print("FAIL", e, batch)
+        print("FAIL", e, batch)  # noqa: T201
         self.failed = True
 
     def setUp(self):
@@ -113,8 +112,18 @@ class TestClient(unittest.TestCase):
         }
 
         client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, personal_api_key=FAKE_TEST_API_KEY)
-        variants = client._get_active_feature_variants("some_id", None, None, None)
+        variants = client._get_active_feature_variants("some_id", None, None, None, False)
         self.assertEqual(variants, {"beta-feature": "random-variant", "alpha-feature": True})
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="some_id",
+            groups={},
+            person_properties=None,
+            group_properties=None,
+            disable_geoip=False,
+        )
 
     @mock.patch("posthog.client.decide")
     def test_basic_capture_with_feature_flags_returns_active_only(self, patch_decide):
@@ -132,6 +141,7 @@ class TestClient(unittest.TestCase):
         self.assertTrue(isinstance(msg["timestamp"], str))
         self.assertIsNone(msg.get("uuid"))
         self.assertEqual(msg["distinct_id"], "distinct_id")
+        self.assertTrue(msg["properties"]["$geoip_disable"])
         self.assertEqual(msg["properties"]["$lib"], "posthog-python")
         self.assertEqual(msg["properties"]["$lib_version"], VERSION)
         self.assertEqual(msg["properties"]["$feature/beta-feature"], "random-variant")
@@ -139,6 +149,53 @@ class TestClient(unittest.TestCase):
         self.assertEqual(msg["properties"]["$active_feature_flags"], ["beta-feature", "alpha-feature"])
 
         self.assertEqual(patch_decide.call_count, 1)
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="distinct_id",
+            groups={},
+            person_properties=None,
+            group_properties=None,
+            disable_geoip=True,
+        )
+
+    @mock.patch("posthog.client.decide")
+    def test_basic_capture_with_feature_flags_and_disable_geoip_returns_correctly(self, patch_decide):
+        patch_decide.return_value = {
+            "featureFlags": {"beta-feature": "random-variant", "alpha-feature": True, "off-feature": False}
+        }
+
+        client = Client(
+            FAKE_TEST_API_KEY, on_error=self.set_fail, personal_api_key=FAKE_TEST_API_KEY, disable_geoip=True
+        )
+        success, msg = client.capture("distinct_id", "python test event", send_feature_flags=True, disable_geoip=False)
+        client.flush()
+        self.assertTrue(success)
+        self.assertFalse(self.failed)
+
+        self.assertEqual(msg["event"], "python test event")
+        self.assertTrue(isinstance(msg["timestamp"], str))
+        self.assertIsNone(msg.get("uuid"))
+        self.assertTrue("$geoip_disable" not in msg["properties"])
+        self.assertEqual(msg["distinct_id"], "distinct_id")
+        self.assertEqual(msg["properties"]["$lib"], "posthog-python")
+        self.assertEqual(msg["properties"]["$lib_version"], VERSION)
+        self.assertEqual(msg["properties"]["$feature/beta-feature"], "random-variant")
+        self.assertEqual(msg["properties"]["$feature/alpha-feature"], True)
+        self.assertEqual(msg["properties"]["$active_feature_flags"], ["beta-feature", "alpha-feature"])
+
+        self.assertEqual(patch_decide.call_count, 1)
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="distinct_id",
+            groups={},
+            person_properties=None,
+            group_properties=None,
+            disable_geoip=False,
+        )
 
     @mock.patch("posthog.client.decide")
     def test_basic_capture_with_feature_flags_switched_off_doesnt_send_them(self, patch_decide):
@@ -306,6 +363,7 @@ class TestClient(unittest.TestCase):
                 "$group_set": {},
                 "$lib": "posthog-python",
                 "$lib_version": VERSION,
+                "$geoip_disable": True,
             },
         )
         self.assertTrue(isinstance(msg["timestamp"], str))
@@ -327,6 +385,7 @@ class TestClient(unittest.TestCase):
                 "$group_set": {"trait": "value"},
                 "$lib": "posthog-python",
                 "$lib_version": VERSION,
+                "$geoip_disable": True,
             },
         )
         self.assertEqual(msg["timestamp"], "2014-09-03T00:00:00+00:00")
@@ -469,6 +528,127 @@ class TestClient(unittest.TestCase):
         for consumer in client.consumers:
             self.assertEqual(consumer.timeout, 15)
 
+    def test_disabled(self):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disabled=True)
+        success, msg = client.capture("distinct_id", "python test event")
+        client.flush()
+        self.assertFalse(success)
+        self.assertFalse(self.failed)
+
+        self.assertEqual(msg, "disabled")
+
+    @mock.patch("posthog.client.decide")
+    def test_disabled_with_feature_flags(self, patch_decide):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disabled=True)
+
+        response = client.get_feature_flag("beta-feature", "12345")
+        self.assertIsNone(response)
+        patch_decide.assert_not_called()
+
+        response = client.feature_enabled("beta-feature", "12345")
+        self.assertIsNone(response)
+        patch_decide.assert_not_called()
+
+        response = client.get_all_flags("12345")
+        self.assertIsNone(response)
+        patch_decide.assert_not_called()
+
+        response = client.get_feature_flag_payload("key", "12345")
+        self.assertIsNone(response)
+        patch_decide.assert_not_called()
+
+        response = client.get_all_flags_and_payloads("12345")
+        self.assertEqual(response, {"featureFlags": None, "featureFlagPayloads": None})
+        patch_decide.assert_not_called()
+
+        # no capture calls
+        self.assertTrue(client.queue.empty())
+
+    def test_enabled_to_disabled(self):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disabled=False)
+        success, msg = client.capture("distinct_id", "python test event")
+        client.flush()
+
+        self.assertTrue(success)
+        self.assertFalse(self.failed)
+        self.assertEqual(msg["event"], "python test event")
+
+        client.disabled = True
+        success, msg = client.capture("distinct_id", "python test event")
+        client.flush()
+        self.assertFalse(success)
+        self.assertFalse(self.failed)
+
+        self.assertEqual(msg, "disabled")
+
+    def test_disable_geoip_default_on_events(self):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disable_geoip=True)
+        _, capture_msg = client.capture("distinct_id", "python test event")
+        client.flush()
+        self.assertEqual(capture_msg["properties"]["$geoip_disable"], True)
+
+        _, identify_msg = client.identify("distinct_id", {"trait": "value"})
+        client.flush()
+        self.assertEqual(identify_msg["properties"]["$geoip_disable"], True)
+
+    def test_disable_geoip_override_on_events(self):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disable_geoip=False)
+        _, capture_msg = client.set("distinct_id", {"a": "b", "c": "d"}, disable_geoip=True)
+        client.flush()
+        self.assertEqual(capture_msg["properties"]["$geoip_disable"], True)
+
+        _, identify_msg = client.page("distinct_id", "http://a.com", {"trait": "value"}, disable_geoip=False)
+        client.flush()
+        self.assertEqual("$geoip_disable" not in identify_msg["properties"], True)
+
+    def test_disable_geoip_method_overrides_init_on_events(self):
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disable_geoip=True)
+        _, msg = client.capture("distinct_id", "python test event", disable_geoip=False)
+        client.flush()
+        self.assertTrue("$geoip_disable" not in msg["properties"])
+
+    @mock.patch("posthog.client.decide")
+    def test_disable_geoip_default_on_decide(self, patch_decide):
+        patch_decide.return_value = {
+            "featureFlags": {"beta-feature": "random-variant", "alpha-feature": True, "off-feature": False}
+        }
+        client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, disable_geoip=False)
+        client.get_feature_flag("random_key", "some_id", disable_geoip=True)
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="some_id",
+            groups={},
+            person_properties={},
+            group_properties={},
+            disable_geoip=True,
+        )
+        patch_decide.reset_mock()
+        client.feature_enabled("random_key", "feature_enabled_distinct_id", disable_geoip=True)
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="feature_enabled_distinct_id",
+            groups={},
+            person_properties={},
+            group_properties={},
+            disable_geoip=True,
+        )
+        patch_decide.reset_mock()
+        client.get_all_flags_and_payloads("all_flags_payloads_id")
+        patch_decide.assert_called_with(
+            "random_key",
+            None,
+            timeout=10,
+            distinct_id="all_flags_payloads_id",
+            groups={},
+            person_properties={},
+            group_properties={},
+            disable_geoip=False,
+        )
+
     @mock.patch("posthog.client.Poller")
     @mock.patch("posthog.client.get")
     def test_call_identify_fails(self, patch_get, patch_poll):

posthog/test/test_consumer.py

@@ -145,15 +145,20 @@ class TestConsumer(unittest.TestCase):
     def test_max_batch_size(self):
         q = Queue()
         consumer = Consumer(q, TEST_API_KEY, flush_at=100000, flush_interval=3)
-        track = {"type": "track", "event": "python event", "distinct_id": "distinct_id"}
+        properties = {}
+        for n in range(0, 500):
+            properties[str(n)] = "one_long_property_value_to_build_a_big_event"
+        track = {"type": "track", "event": "python event", "distinct_id": "distinct_id", "properties": properties}
         msg_size = len(json.dumps(track).encode())
-        # number of messages in a maximum-size batch
-        n_msgs = int(475000 / msg_size)
+        # Let's capture 8MB of data to trigger two batches
+        n_msgs = int(8_000_000 / msg_size)
 
         def mock_post_fn(_, data, **kwargs):
             res = mock.Mock()
             res.status_code = 200
-            self.assertTrue(len(data.encode()) < 500000, "batch size (%d) exceeds 500KB limit" % len(data.encode()))
+            request_size = len(data.encode())
+            # Batches close after the first message bringing it bigger than BATCH_SIZE_LIMIT, let's add 10% of margin
+            self.assertTrue(request_size < (5 * 1024 * 1024) * 1.1, "batch size (%d) higher than limit" % request_size)
             return res
 
         with mock.patch("posthog.request._session.post", side_effect=mock_post_fn) as mock_post:

posthog/test/test_feature_flags.py

@@ -24,7 +24,7 @@ class TestLocalEvaluation(unittest.TestCase):
 
     def set_fail(self, e, batch):
         """Mark the failure handler"""
-        print("FAIL", e, batch)
+        print("FAIL", e, batch)  # noqa: T201
         self.failed = True
 
     def setUp(self):
@@ -1739,7 +1739,7 @@ class TestMatchProperties(unittest.TestCase):
         self.assertFalse(match_property(property_b, {"key": "three"}))
 
     def test_match_properties_regex(self):
-        property_a = self.property(key="key", value="\.com$", operator="regex")
+        property_a = self.property(key="key", value="\.com$", operator="regex")  # noqa: W605
         self.assertTrue(match_property(property_a, {"key": "value.com"}))
         self.assertTrue(match_property(property_a, {"key": "value2.com"}))
 
@@ -1747,7 +1747,7 @@ class TestMatchProperties(unittest.TestCase):
         self.assertFalse(match_property(property_a, {"key": "Alakazam"}))
         self.assertFalse(match_property(property_a, {"key": 123}))
         self.assertFalse(match_property(property_a, {"key": "valuecom"}))
-        self.assertFalse(match_property(property_a, {"key": "value\com"}))
+        self.assertFalse(match_property(property_a, {"key": "value\com"}))  # noqa: W605
 
         property_b = self.property(key="key", value="3", operator="regex")
         self.assertTrue(match_property(property_b, {"key": "3"}))
@@ -1888,8 +1888,14 @@ class TestCaptureCalls(unittest.TestCase):
         patch_capture.assert_called_with(
             "some-distinct-id",
             "$feature_flag_called",
-            {"$feature_flag": "complex-flag", "$feature_flag_response": True, "locally_evaluated": True},
+            {
+                "$feature_flag": "complex-flag",
+                "$feature_flag_response": True,
+                "locally_evaluated": True,
+                "$feature/complex-flag": True,
+            },
             groups={},
+            disable_geoip=None,
         )
         patch_capture.reset_mock()
 
@@ -1912,8 +1918,14 @@ class TestCaptureCalls(unittest.TestCase):
         patch_capture.assert_called_with(
             "some-distinct-id2",
             "$feature_flag_called",
-            {"$feature_flag": "complex-flag", "$feature_flag_response": True, "locally_evaluated": True},
+            {
+                "$feature_flag": "complex-flag",
+                "$feature_flag_response": True,
+                "locally_evaluated": True,
+                "$feature/complex-flag": True,
+            },
             groups={},
+            disable_geoip=None,
         )
         patch_capture.reset_mock()
 
@@ -1944,8 +1956,57 @@ class TestCaptureCalls(unittest.TestCase):
         patch_capture.assert_called_with(
             "some-distinct-id2",
             "$feature_flag_called",
-            {"$feature_flag": "decide-flag", "$feature_flag_response": "decide-value", "locally_evaluated": False},
+            {
+                "$feature_flag": "decide-flag",
+                "$feature_flag_response": "decide-value",
+                "locally_evaluated": False,
+                "$feature/decide-flag": "decide-value",
+            },
             groups={"organization": "org1"},
+            disable_geoip=None,
+        )
+
+    @mock.patch.object(Client, "capture")
+    @mock.patch("posthog.client.decide")
+    def test_disable_geoip_get_flag_capture_call(self, patch_decide, patch_capture):
+        patch_decide.return_value = {"featureFlags": {"decide-flag": "decide-value"}}
+        client = Client(FAKE_TEST_API_KEY, personal_api_key=FAKE_TEST_API_KEY, disable_geoip=True)
+        client.feature_flags = [
+            {
+                "id": 1,
+                "name": "Beta Feature",
+                "key": "complex-flag",
+                "is_simple_flag": False,
+                "active": True,
+                "filters": {
+                    "groups": [
+                        {
+                            "properties": [{"key": "region", "value": "USA"}],
+                            "rollout_percentage": 100,
+                        }
+                    ],
+                },
+            }
+        ]
+
+        client.get_feature_flag(
+            "complex-flag",
+            "some-distinct-id",
+            person_properties={"region": "USA", "name": "Aloha"},
+            disable_geoip=False,
+        )
+
+        patch_capture.assert_called_with(
+            "some-distinct-id",
+            "$feature_flag_called",
+            {
+                "$feature_flag": "complex-flag",
+                "$feature_flag_response": True,
+                "locally_evaluated": True,
+                "$feature/complex-flag": True,
+            },
+            groups={},
+            disable_geoip=False,
         )
 
     @mock.patch("posthog.client.MAX_DICT_SIZE", 100)
@@ -1977,8 +2038,14 @@ class TestCaptureCalls(unittest.TestCase):
             patch_capture.assert_called_with(
                 distinct_id,
                 "$feature_flag_called",
-                {"$feature_flag": "complex-flag", "$feature_flag_response": True, "locally_evaluated": True},
+                {
+                    "$feature_flag": "complex-flag",
+                    "$feature_flag_response": True,
+                    "locally_evaluated": True,
+                    "$feature/complex-flag": True,
+                },
                 groups={},
+                disable_geoip=None,
             )
 
             self.assertEqual(len(client.distinct_ids_feature_flags_reported), i % 100 + 1)
@@ -1997,7 +2064,7 @@ class TestConsistency(unittest.TestCase):
 
     def set_fail(self, e, batch):
         """Mark the failure handler"""
-        print("FAIL", e, batch)
+        print("FAIL", e, batch)  # noqa: T201
         self.failed = True
 
     def setUp(self):

posthog/test/test_module.py

@@ -1,40 +1,41 @@
 import unittest
 
-import posthog
+from posthog import Posthog
 
 
 class TestModule(unittest.TestCase):
+    posthog = None
+
     def failed(self):
         self.failed = True
 
     def setUp(self):
         self.failed = False
-        posthog.api_key = "testsecret"
-        posthog.on_error = self.failed
+        self.posthog = Posthog("testsecret", host="http://localhost:8000", on_error=self.failed)
 
     def test_no_api_key(self):
-        posthog.api_key = None
-        self.assertRaises(Exception, posthog.capture)
+        self.posthog.api_key = None
+        self.assertRaises(Exception, self.posthog.capture)
 
     def test_no_host(self):
-        posthog.host = None
-        self.assertRaises(Exception, posthog.capture)
+        self.posthog.host = None
+        self.assertRaises(Exception, self.posthog.capture)
 
     def test_track(self):
-        posthog.capture("distinct_id", "python module event")
-        posthog.flush()
+        self.posthog.capture("distinct_id", "python module event")
+        self.posthog.flush()
 
     def test_identify(self):
-        posthog.identify("distinct_id", {"email": "user@email.com"})
-        posthog.flush()
+        self.posthog.identify("distinct_id", {"email": "user@email.com"})
+        self.posthog.flush()
 
     def test_alias(self):
-        posthog.alias("previousId", "distinct_id")
-        posthog.flush()
+        self.posthog.alias("previousId", "distinct_id")
+        self.posthog.flush()
 
     def test_page(self):
-        posthog.page("distinct_id", "https://posthog.com/contact")
-        posthog.flush()
+        self.posthog.page("distinct_id", "https://posthog.com/contact")
+        self.posthog.flush()
 
     def test_flush(self):
-        posthog.flush()
+        self.posthog.flush()

posthog/version.py

@@ -1,4 +1,4 @@
-VERSION = "2.4.0"
+VERSION = "3.1.0"
 
 if __name__ == "__main__":
-    print(VERSION, end="")
+    print(VERSION, end="")  # noqa: T201

setup.py

@@ -20,9 +20,11 @@ extras_require = {
     "dev": [
         "black",
         "isort",
+        "flake8",
+        "flake8-print",
         "pre-commit",
     ],
-    "test": ["mock>=2.0.0", "freezegun==0.3.15", "pylint", "flake8", "coverage", "pytest"],
+    "test": ["mock>=2.0.0", "freezegun==0.3.15", "pylint", "flake8", "coverage", "pytest", "pytest-timeout"],
     "sentry": ["sentry-sdk", "django"],
 }
 

setup_analytics.py

@@ -14,7 +14,7 @@ long_description = """
 PostHog is developer-friendly, self-hosted product analytics. posthog-python is the python package.
 """
 
-install_requires = ["requests>=2.7,<3.0", "six>=1.5", "monotonic>=1.5", "backoff==1.6.0", "python-dateutil>2.1"]
+install_requires = ["requests>=2.7,<3.0", "six>=1.5", "monotonic>=1.5", "backoff>=1.10.0", "python-dateutil>2.1"]
 
 tests_require = ["mock>=2.0.0"]