feat: add super properties (#138)

Co-authored-by: Neil Kakkar <neilkakkar@gmail.com>

CHANGELOG.md

@@ -1,3 +1,27 @@
+## 3.7.0 - 2024-10-03
+
+1. Adds a new `super_properties` parameter on the client that are appended to every /capture call.
+
+## 3.6.7 - 2024-09-24
+
+1. Remove deprecated datetime.utcnow() in favour of datetime.now(tz=tzutc())
+
+## 3.6.6 - 2024-09-16
+
+1. Fix manual capture support for in app frames
+
+## 3.6.5 - 2024-09-10
+
+1. Fix django integration support for manual exception capture.
+
+## 3.6.4 - 2024-09-05
+
+1. Add manual exception capture.
+
+## 3.6.3 - 2024-09-03
+
+1. Make sure setup.py for posthoganalytics package also discovers the new exception integration package.
+
 ## 3.6.2 - 2024-09-03
 
 1. Make sure setup.py discovers the new exception integration package.
@@ -46,7 +70,6 @@
 
 1. Return success/failure with all capture calls from module functions
 
-
 ## 3.3.1 - 2024-01-10
 
 1. Make sure we don't override any existing feature flag properties when adding locally evaluated feature flag properties.
@@ -72,6 +95,7 @@
 
 1. Restore how feature flags work when the client library is disabled: All requests return `None` and no events are sent when the client is disabled.
 2. Add a `feature_flag_definitions()` debug option, which returns currently loaded feature flag definitions. You can use this to more cleverly decide when to request local evaluation of feature flags.
+
 ## 3.0.0 - 2023-04-14
 
 Breaking change:
@@ -112,7 +136,6 @@ posthog = Posthog('api_key', disable_geoip=False)
 1. Log instead of raise error on posthog personal api key errors
 2. Remove upper bound on backoff dependency
 
-
 ## 2.3.0 - 2023-01-31
 
 1. Add support for returning payloads of matched feature flags
@@ -128,6 +151,7 @@ Changes:
 Changes:
 
 1. Fixes issues with date comparison.
+
 ## 2.1.1 - 2022-09-14
 
 Changes:
@@ -140,6 +164,7 @@ Changes:
 
 1. Feature flag defaults have been removed
 2. Setup logging only when debug mode is enabled.
+
 ## 2.0.1 - 2022-08-04
 
 - Make poll_interval configurable
@@ -152,7 +177,7 @@ Breaking changes:
 
 1. The minimum version requirement for PostHog servers is now 1.38. If you're using PostHog Cloud, you satisfy this requirement automatically.
 2. Feature flag defaults apply only when there's an error fetching feature flag results. Earlier, if the default was set to `True`, even if a flag resolved to `False`, the default would override this.
-**Note: These are removed in 2.0.2**
+   **Note: These are removed in 2.0.2**
 3. Feature flag remote evaluation doesn't require a personal API key.
 
 New Changes:
@@ -160,18 +185,20 @@ New Changes:
 1. You can now evaluate feature flags locally (i.e. without sending a request to your PostHog servers) by setting a personal API key, and passing in groups and person properties to `is_feature_enabled` and `get_feature_flag` calls.
 2. Introduces a `get_all_flags` method that returns all feature flags. This is useful for when you want to seed your frontend with some initial flags, given a user ID.
 
-
-
 ## 1.4.9 - 2022-06-13
+
 - Support for sending feature flags with capture calls
 
 ## 1.4.8 - 2022-05-12
+
 - Support multi variate feature flags
 
 ## 1.4.7 - 2022-04-25
+
 - Allow feature flags usage without project_api_key
 
 ## 1.4.1 - 2021-05-28
+
 - Fix packaging issues with Sentry integrations
 
 ## 1.4.0 - 2021-05-18

posthog/__init__.py

@@ -2,7 +2,7 @@ import datetime  # noqa: F401
 from typing import Callable, Dict, List, Optional, Tuple  # noqa: F401
 
 from posthog.client import Client
-from posthog.exception_capture import Integrations  # noqa: F401
+from posthog.exception_capture import DEFAULT_DISTINCT_ID, Integrations  # noqa: F401
 from posthog.version import VERSION
 
 __version__ = VERSION
@@ -20,9 +20,12 @@ project_api_key = None  # type: Optional[str]
 poll_interval = 30  # type: int
 disable_geoip = True  # type: bool
 feature_flags_request_timeout_seconds = 3  # type: int
+super_properties = None  # type: Optional[Dict]
 # Currently alpha, use at your own risk
 enable_exception_autocapture = False  # type: bool
 exception_autocapture_integrations = []  # type: List[Integrations]
+# Used to determine in app paths for exception autocapture. Defaults to the current working directory
+project_root = None  # type: Optional[str]
 
 default_client = None  # type: Optional[Client]
 
@@ -251,6 +254,50 @@ def alias(
     )
 
 
+def capture_exception(
+    exception=None,  # type: Optional[BaseException]
+    distinct_id=None,  # type: Optional[str]
+    properties=None,  # type: Optional[Dict]
+    context=None,  # type: Optional[Dict]
+    timestamp=None,  # type: Optional[datetime.datetime]
+    uuid=None,  # type: Optional[str]
+    groups=None,  # type: Optional[Dict]
+):
+    # type: (...) -> Tuple[bool, dict]
+    """
+    capture_exception allows you to capture exceptions that happen in your code. This is useful for debugging and understanding what errors your users are encountering.
+    This function never raises an exception, even if it fails to send the event.
+
+    A `capture_exception` call does not require any fields, but we recommend sending:
+    - `distinct id` which uniquely identifies your user for which this exception happens
+    - `exception` to specify the exception to capture. If not provided, the current exception is captured via `sys.exc_info()`
+
+    Optionally you can submit
+    - `properties`, which can be a dict with any information you'd like to add
+    - `groups`, which is a dict of group type -> group key mappings
+
+    For example:
+    ```python
+    try:
+        1 / 0
+    except Exception as e:
+        posthog.capture_exception(e, 'my specific distinct id')
+        posthog.capture_exception(distinct_id='my specific distinct id')
+
+    ```
+    """
+    return _proxy(
+        "capture_exception",
+        exception=exception,
+        distinct_id=distinct_id or DEFAULT_DISTINCT_ID,
+        properties=properties,
+        context=context,
+        timestamp=timestamp,
+        uuid=uuid,
+        groups=groups,
+    )
+
+
 def feature_enabled(
     key,  # type: str
     distinct_id,  # type: str
@@ -458,6 +505,7 @@ def _proxy(method, *args, **kwargs):
             disabled=disabled,
             disable_geoip=disable_geoip,
             feature_flags_request_timeout_seconds=feature_flags_request_timeout_seconds,
+            super_properties=super_properties,
             # TODO: Currently this monitoring begins only when the Client is initialised (which happens when you do something with the SDK)
             # This kind of initialisation is very annoying for exception capture. We need to figure out a way around this,
             # or deprecate this proxy option fully (it's already in the process of deprecation, no new clients should be using this method since like 5-6 months)

posthog/client.py

@@ -1,6 +1,8 @@
 import atexit
 import logging
 import numbers
+import os
+import sys
 from datetime import datetime, timedelta
 from uuid import UUID
 
@@ -8,11 +10,12 @@ from dateutil.tz import tzutc
 from six import string_types
 
 from posthog.consumer import Consumer
-from posthog.exception_capture import ExceptionCapture
+from posthog.exception_capture import DEFAULT_DISTINCT_ID, ExceptionCapture
+from posthog.exception_utils import exc_info_from_error, exceptions_from_error_tuple, handle_in_app
 from posthog.feature_flags import InconclusiveMatchError, match_feature_flag_properties
 from posthog.poller import Poller
-from posthog.request import APIError, batch_post, decide, determine_server_host, get
-from posthog.utils import SizeLimitedDict, clean, guess_timezone
+from posthog.request import DEFAULT_HOST, APIError, batch_post, decide, determine_server_host, get
+from posthog.utils import SizeLimitedDict, clean, guess_timezone, remove_trailing_slash
 from posthog.version import VERSION
 
 try:
@@ -52,8 +55,10 @@ class Client(object):
         disable_geoip=True,
         historical_migration=False,
         feature_flags_request_timeout_seconds=3,
+        super_properties=None,
         enable_exception_autocapture=False,
         exception_autocapture_integrations=None,
+        project_root=None,
     ):
         self.queue = queue.Queue(max_queue_size)
 
@@ -67,7 +72,7 @@ class Client(object):
         self.send = send
         self.sync_mode = sync_mode
         # Used for session replay URL generation - we don't want the server host here.
-        self.raw_host = host
+        self.raw_host = host or DEFAULT_HOST
         self.host = determine_server_host(host)
         self.gzip = gzip
         self.timeout = timeout
@@ -82,10 +87,19 @@ class Client(object):
         self.disabled = disabled
         self.disable_geoip = disable_geoip
         self.historical_migration = historical_migration
+        self.super_properties = super_properties
         self.enable_exception_autocapture = enable_exception_autocapture
         self.exception_autocapture_integrations = exception_autocapture_integrations
         self.exception_capture = None
 
+        if project_root is None:
+            try:
+                project_root = os.getcwd()
+            except Exception:
+                project_root = None
+
+        self.project_root = project_root
+
         # personal_api_key: This should be a generated Personal API Key, private
         self.personal_api_key = personal_api_key
         if debug:
@@ -345,6 +359,58 @@ class Client(object):
 
         return self._enqueue(msg, disable_geoip)
 
+    def capture_exception(
+        self,
+        exception=None,
+        distinct_id=DEFAULT_DISTINCT_ID,
+        properties=None,
+        context=None,
+        timestamp=None,
+        uuid=None,
+        groups=None,
+    ):
+        # this function shouldn't ever throw an error, so it logs exceptions instead of raising them.
+        # this is important to ensure we don't unexpectedly re-raise exceptions in the user's code.
+        try:
+            properties = properties or {}
+            require("distinct_id", distinct_id, ID_TYPES)
+            require("properties", properties, dict)
+
+            if exception is not None:
+                exc_info = exc_info_from_error(exception)
+            else:
+                exc_info = sys.exc_info()
+
+            if exc_info is None or exc_info == (None, None, None):
+                self.log.warning("No exception information available")
+                return
+
+            # Format stack trace like sentry
+            all_exceptions_with_trace = exceptions_from_error_tuple(exc_info)
+
+            # Add in-app property to frames in the exceptions
+            event = handle_in_app(
+                {
+                    "exception": {
+                        "values": all_exceptions_with_trace,
+                    },
+                },
+                project_root=self.project_root,
+            )
+            all_exceptions_with_trace_and_in_app = event["exception"]["values"]
+
+            properties = {
+                "$exception_type": all_exceptions_with_trace_and_in_app[0].get("type"),
+                "$exception_message": all_exceptions_with_trace_and_in_app[0].get("value"),
+                "$exception_list": all_exceptions_with_trace_and_in_app,
+                "$exception_personURL": f"{remove_trailing_slash(self.raw_host)}/project/{self.api_key}/person/{distinct_id}",
+                **properties,
+            }
+
+            return self.capture(distinct_id, "$exception", properties, context, timestamp, uuid, groups)
+        except Exception as e:
+            self.log.exception(f"Failed to capture exception: {e}")
+
     def _enqueue(self, msg, disable_geoip):
         """Push a new `msg` onto the queue, return `(success, msg)`"""
 
@@ -353,7 +419,7 @@ class Client(object):
 
         timestamp = msg["timestamp"]
         if timestamp is None:
-            timestamp = datetime.utcnow().replace(tzinfo=tzutc())
+            timestamp = datetime.now(tz=tzutc())
 
         require("timestamp", timestamp, datetime)
         require("context", msg["context"], dict)
@@ -379,6 +445,9 @@ class Client(object):
         if disable_geoip:
             msg["properties"]["$geoip_disable"] = True
 
+        if self.super_properties:
+            msg["properties"] = {**msg["properties"], **self.super_properties}
+
         msg["distinct_id"] = stringify_id(msg.get("distinct_id", None))
 
         msg = clean(msg)
@@ -477,7 +546,7 @@ class Client(object):
             )
             self.log.warning(e)
 
-        self._last_feature_flag_poll = datetime.utcnow().replace(tzinfo=tzutc())
+        self._last_feature_flag_poll = datetime.now(tz=tzutc())
 
     def load_feature_flags(self):
         if not self.personal_api_key:

posthog/exception_capture.py

@@ -4,9 +4,6 @@ import threading
 from enum import Enum
 from typing import TYPE_CHECKING, List, Optional
 
-from posthog.exception_utils import exceptions_from_error_tuple, handle_in_app
-from posthog.utils import remove_trailing_slash
-
 if TYPE_CHECKING:
     from posthog.client import Client
 
@@ -49,52 +46,29 @@ class ExceptionCapture:
 
     def exception_handler(self, exc_type, exc_value, exc_traceback):
         # don't affect default behaviour.
-        self.capture_exception(exc_type, exc_value, exc_traceback)
+        self.capture_exception((exc_type, exc_value, exc_traceback))
         self.original_excepthook(exc_type, exc_value, exc_traceback)
 
     def thread_exception_handler(self, args):
-        self.capture_exception(args.exc_type, args.exc_value, args.exc_traceback)
+        self.capture_exception((args.exc_type, args.exc_value, args.exc_traceback))
 
     def exception_receiver(self, exc_info, extra_properties):
         if "distinct_id" in extra_properties:
             metadata = {"distinct_id": extra_properties["distinct_id"]}
         else:
             metadata = None
-        self.capture_exception(exc_info[0], exc_info[1], exc_info[2], metadata)
+        self.capture_exception((exc_info[0], exc_info[1], exc_info[2]), metadata)
 
-    def capture_exception(self, exc_type, exc_value, exc_traceback, metadata=None):
+    def capture_exception(self, exception, metadata=None):
         try:
             # if hasattr(sys, "ps1"):
             #     # Disable the excepthook for interactive Python shells
             #     return
 
-            # Format stack trace like sentry
-            all_exceptions_with_trace = exceptions_from_error_tuple((exc_type, exc_value, exc_traceback))
-
-            # Add in-app property to frames in the exceptions
-            event = handle_in_app(
-                {
-                    "exception": {
-                        "values": all_exceptions_with_trace,
-                    },
-                }
-            )
-            all_exceptions_with_trace_and_in_app = event["exception"]["values"]
-
             distinct_id = metadata.get("distinct_id") if metadata else DEFAULT_DISTINCT_ID
             # Make sure we have a distinct_id if its empty in metadata
             distinct_id = distinct_id or DEFAULT_DISTINCT_ID
 
-            properties = {
-                "$exception_type": all_exceptions_with_trace_and_in_app[0].get("type"),
-                "$exception_message": all_exceptions_with_trace_and_in_app[0].get("value"),
-                "$exception_list": all_exceptions_with_trace_and_in_app,
-                "$exception_personURL": f"{remove_trailing_slash(self.client.raw_host)}/project/{self.client.api_key}/person/{distinct_id}",
-            }
-
-            # TODO: What distinct id should we attach these server-side exceptions to?
-            # Any heuristic seems prone to errors - how can we know if exception occurred in the context of a user that captured some other event?
-
-            self.client.capture(distinct_id, "$exception", properties=properties)
+            self.client.capture_exception(exception, distinct_id)
         except Exception as e:
             self.log.exception(f"Failed to capture exception: {e}")

posthog/request.py

@@ -37,7 +37,7 @@ def post(
     """Post the `kwargs` to the API"""
     log = logging.getLogger("posthog")
     body = kwargs
-    body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat()
+    body["sentAt"] = datetime.now(tz=tzutc()).isoformat()
     url = remove_trailing_slash(host or DEFAULT_HOST) + path
     body["api_key"] = api_key
     data = json.dumps(body, cls=DatetimeSerializer)

posthog/test/test_client.py

@@ -84,6 +84,180 @@ class TestClient(unittest.TestCase):
         self.assertEqual(msg["properties"]["$lib"], "posthog-python")
         self.assertEqual(msg["properties"]["$lib_version"], VERSION)
 
+    def test_basic_super_properties(self):
+        client = Client(FAKE_TEST_API_KEY, super_properties={"source": "repo-name"})
+
+        _, msg = client.capture("distinct_id", "python test event")
+        client.flush()
+
+        self.assertEqual(msg["event"], "python test event")
+        self.assertEqual(msg["properties"]["source"], "repo-name")
+
+        _, msg = client.identify("distinct_id", {"trait": "value"})
+        client.flush()
+
+        self.assertEqual(msg["$set"]["trait"], "value")
+        self.assertEqual(msg["properties"]["source"], "repo-name")
+
+    def test_basic_capture_exception(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            client = self.client
+            exception = Exception("test exception")
+            client.capture_exception(exception)
+
+            self.assertTrue(patch_capture.called)
+            capture_call = patch_capture.call_args[0]
+            self.assertEqual(capture_call[0], "python-exceptions")
+            self.assertEqual(capture_call[1], "$exception")
+            self.assertEqual(
+                capture_call[2],
+                {
+                    "$exception_type": "Exception",
+                    "$exception_message": "test exception",
+                    "$exception_list": [
+                        {
+                            "mechanism": {"type": "generic", "handled": True},
+                            "module": None,
+                            "type": "Exception",
+                            "value": "test exception",
+                        }
+                    ],
+                    "$exception_personURL": "https://us.i.posthog.com/project/random_key/person/python-exceptions",
+                },
+            )
+
+    def test_basic_capture_exception_with_distinct_id(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            client = self.client
+            exception = Exception("test exception")
+            client.capture_exception(exception, "distinct_id")
+
+            self.assertTrue(patch_capture.called)
+            capture_call = patch_capture.call_args[0]
+            self.assertEqual(capture_call[0], "distinct_id")
+            self.assertEqual(capture_call[1], "$exception")
+            self.assertEqual(
+                capture_call[2],
+                {
+                    "$exception_type": "Exception",
+                    "$exception_message": "test exception",
+                    "$exception_list": [
+                        {
+                            "mechanism": {"type": "generic", "handled": True},
+                            "module": None,
+                            "type": "Exception",
+                            "value": "test exception",
+                        }
+                    ],
+                    "$exception_personURL": "https://us.i.posthog.com/project/random_key/person/distinct_id",
+                },
+            )
+
+    def test_basic_capture_exception_with_correct_host_generation(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, host="https://aloha.com")
+            exception = Exception("test exception")
+            client.capture_exception(exception, "distinct_id")
+
+            self.assertTrue(patch_capture.called)
+            capture_call = patch_capture.call_args[0]
+            self.assertEqual(capture_call[0], "distinct_id")
+            self.assertEqual(capture_call[1], "$exception")
+            self.assertEqual(
+                capture_call[2],
+                {
+                    "$exception_type": "Exception",
+                    "$exception_message": "test exception",
+                    "$exception_list": [
+                        {
+                            "mechanism": {"type": "generic", "handled": True},
+                            "module": None,
+                            "type": "Exception",
+                            "value": "test exception",
+                        }
+                    ],
+                    "$exception_personURL": "https://aloha.com/project/random_key/person/distinct_id",
+                },
+            )
+
+    def test_basic_capture_exception_with_correct_host_generation_for_server_hosts(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            client = Client(FAKE_TEST_API_KEY, on_error=self.set_fail, host="https://app.posthog.com")
+            exception = Exception("test exception")
+            client.capture_exception(exception, "distinct_id")
+
+            self.assertTrue(patch_capture.called)
+            capture_call = patch_capture.call_args[0]
+            self.assertEqual(capture_call[0], "distinct_id")
+            self.assertEqual(capture_call[1], "$exception")
+            self.assertEqual(
+                capture_call[2],
+                {
+                    "$exception_type": "Exception",
+                    "$exception_message": "test exception",
+                    "$exception_list": [
+                        {
+                            "mechanism": {"type": "generic", "handled": True},
+                            "module": None,
+                            "type": "Exception",
+                            "value": "test exception",
+                        }
+                    ],
+                    "$exception_personURL": "https://app.posthog.com/project/random_key/person/distinct_id",
+                },
+            )
+
+    def test_basic_capture_exception_with_no_exception_given(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            client = self.client
+            try:
+                raise Exception("test exception")
+            except Exception:
+                client.capture_exception()
+
+            self.assertTrue(patch_capture.called)
+            capture_call = patch_capture.call_args[0]
+            self.assertEqual(capture_call[0], "python-exceptions")
+            self.assertEqual(capture_call[1], "$exception")
+            self.assertEqual(capture_call[2]["$exception_type"], "Exception")
+            self.assertEqual(capture_call[2]["$exception_message"], "test exception")
+            self.assertEqual(capture_call[2]["$exception_list"][0]["mechanism"]["type"], "generic")
+            self.assertEqual(capture_call[2]["$exception_list"][0]["mechanism"]["handled"], True)
+            self.assertEqual(capture_call[2]["$exception_list"][0]["module"], None)
+            self.assertEqual(capture_call[2]["$exception_list"][0]["type"], "Exception")
+            self.assertEqual(capture_call[2]["$exception_list"][0]["value"], "test exception")
+            self.assertEqual(
+                capture_call[2]["$exception_list"][0]["stacktrace"]["frames"][0]["filename"],
+                "posthog/test/test_client.py",
+            )
+            self.assertEqual(
+                capture_call[2]["$exception_list"][0]["stacktrace"]["frames"][0]["function"],
+                "test_basic_capture_exception_with_no_exception_given",
+            )
+            self.assertEqual(
+                capture_call[2]["$exception_list"][0]["stacktrace"]["frames"][0]["module"], "posthog.test.test_client"
+            )
+            self.assertEqual(capture_call[2]["$exception_list"][0]["stacktrace"]["frames"][0]["in_app"], True)
+
+    def test_basic_capture_exception_with_no_exception_happening(self):
+
+        with mock.patch.object(Client, "capture", return_value=None) as patch_capture:
+            with self.assertLogs("posthog", level="WARNING") as logs:
+
+                client = self.client
+                client.capture_exception()
+
+                self.assertFalse(patch_capture.called)
+                self.assertEqual(
+                    logs.output[0],
+                    "WARNING:posthog:No exception information available",
+                )
+
     @mock.patch("posthog.client.decide")
     def test_basic_capture_with_feature_flags(self, patch_decide):
         patch_decide.return_value = {"featureFlags": {"beta-feature": "random-variant"}}

posthog/version.py

@@ -1,4 +1,4 @@
-VERSION = "3.6.2"
+VERSION = "3.7.0"
 
 if __name__ == "__main__":
     print(VERSION, end="")  # noqa: T201

setup_analytics.py

@@ -27,7 +27,12 @@ setup(
     maintainer="PostHog",
     maintainer_email="hey@posthog.com",
     test_suite="posthoganalytics.test.all",
-    packages=["posthoganalytics", "posthoganalytics.test", "posthoganalytics.sentry"],
+    packages=[
+        "posthoganalytics",
+        "posthoganalytics.test",
+        "posthoganalytics.sentry",
+        "posthoganalytics.exception_integrations",
+    ],
     license="MIT License",
     install_requires=install_requires,
     tests_require=tests_require,