WIP: e2e tests framework with pytest

pytest.ini

@@ -1,3 +1,4 @@
 # Handling DeprecationWarning 'asyncio_mode' default value
 [pytest]
 asyncio_mode = strict
+asyncio_default_fixture_loop_scope = function

requirements.txt

@@ -7,5 +7,13 @@ pytest-asyncio
 pytest-rerunfailures
 wheel>=0.38.0
 twine
+testcontainers
 setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability
 zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability
+prometheus_client
+opentelemetry-api>=1.28.2
+opentelemetry-sdk>=1.28.2
+opentelemetry-instrumentation
+opentelemetry-instrumentation-fastapi
+opentelemetry-exporter-otlp
+opentelemetry-exporter-prometheus

tests/.env.example

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export OPAL_POLICY_REPO_URL=''
+export POLICY_REPO_BRANCH=''
+export OPAL_POLICY_REPO_SSH_KEY=''

tests/conftest.py

@@ -0,0 +1,59 @@
+import time
+import docker
+import pytest
+from testcontainers.core.waiting_utils import wait_for_logs
+from testcontainers.postgres import PostgresContainer
+
+from tests.containers import OpalClientContainer, OpalServerContainer
+
+from . import settings as s
+
+
+@pytest.fixture(scope="session")
+def opal_network():
+    client = docker.from_env()
+    network = client.networks.create(s.OPAL_TESTS_NETWORK_NAME, driver="bridge")
+    yield network.name
+    network.remove()
+
+
+@pytest.fixture(scope="session")
+def broadcast_channel(opal_network: str):
+    with PostgresContainer("postgres:alpine", network=opal_network).with_name(
+        f"pytest_opal_brodcast_channel_{s.OPAL_TESTS_UNIQ_ID}"
+    ) as container:
+        yield container
+
+
+@pytest.fixture(scope="session")
+def opal_server(opal_network: str, broadcast_channel: PostgresContainer):
+    opal_broadcast_uri = broadcast_channel.get_connection_url(
+        host=f"{broadcast_channel._name}.{opal_network}", driver=None
+    )
+
+    with OpalServerContainer(network=opal_network).with_env(
+        "OPAL_BROADCAST_URI", opal_broadcast_uri
+    ) as container:
+        container.get_wrapped_container().reload()
+        print(container.get_wrapped_container().id)
+        wait_for_logs(container, "Clone succeeded")
+        yield container
+
+
+@pytest.fixture(scope="session")
+def opal_client(opal_network: str, opal_server: OpalServerContainer):
+    opal_server_url = f"http://{opal_server._name}.{opal_network}:7002"
+
+    with OpalClientContainer(network=opal_network).with_env(
+        "OPAL_SERVER_URL", opal_server_url
+    ) as container:
+        wait_for_logs(container, "")
+        yield container
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup(opal_server, opal_client):
+    yield
+    if s.OPAL_TESTS_DEBUG:
+        s.dump_settings()
+        time.sleep(3600)  # Giving us some time to inspect the containers

tests/containers.py

@@ -0,0 +1,85 @@
+from testcontainers.core.generic import DockerContainer
+
+from . import settings as s
+
+
+class OpalServerContainer(DockerContainer):
+    def __init__(
+        self,
+        image: str = f"permitio/opal-server:{s.OPAL_IMAGE_TAG}",
+        docker_client_kw: dict | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(image, docker_client_kw, **kwargs)
+
+        self.with_name(s.OPAL_TESTS_SERVER_CONTAINER_NAME)
+        self.with_exposed_ports(7002)
+
+        if s.OPAL_TESTS_DEBUG:
+            self.with_env("LOG_DIAGNOSE", "true")
+            self.with_env("OPAL_LOG_LEVEL", "DEBUG")
+
+        self.with_env("UVICORN_NUM_WORKERS", s.UVICORN_NUM_WORKERS)
+
+        self.with_env("OPAL_POLICY_REPO_URL", s.OPAL_POLICY_REPO_URL)
+        self.with_env(
+            "OPAL_POLICY_REPO_POLLING_INTERVAL", s.OPAL_POLICY_REPO_POLLING_INTERVAL
+        )
+        self.with_env("OPAL_POLICY_REPO_MAIN_BRANCH", s.OPAL_POLICY_REPO_MAIN_BRANCH)
+        if s.OPAL_POLICY_REPO_SSH_KEY:
+            self.with_env("OPAL_POLICY_REPO_SSH_KEY", s.OPAL_POLICY_REPO_SSH_KEY)
+        self.with_env(
+            "OPAL_POLICY_REPO_WEBHOOK_SECRET", s.OPAL_POLICY_REPO_WEBHOOK_SECRET
+        )
+        self.with_env(
+            "OPAL_POLICY_REPO_WEBHOOK_PARAMS", s.OPAL_POLICY_REPO_WEBHOOK_PARAMS
+        )
+
+        self.with_env("OPAL_DATA_CONFIG_SOURCES", s.OPAL_DATA_CONFIG_SOURCES)
+        self.with_env("OPAL_LOG_FORMAT_INCLUDE_PID", s.OPAL_LOG_FORMAT_INCLUDE_PID)
+
+        self.with_env("OPAL_AUTH_MASTER_TOKEN", s.OPAL_AUTH_MASTER_TOKEN)
+
+        self.with_env("OPAL_AUTH_PUBLIC_KEY", s.OPAL_AUTH_PUBLIC_KEY)
+        self.with_env("OPAL_AUTH_PRIVATE_KEY", s.OPAL_AUTH_PRIVATE_KEY)
+        if s.OPAL_AUTH_PRIVATE_KEY_PASSPHRASE:
+            self.with_env(
+                "OPAL_AUTH_PRIVATE_KEY_PASSPHRASE", s.OPAL_AUTH_PRIVATE_KEY_PASSPHRASE
+            )
+        self.with_env("OPAL_AUTH_JWT_AUDIENCE", s.OPAL_AUTH_JWT_AUDIENCE)
+        self.with_env("OPAL_AUTH_JWT_ISSUER", s.OPAL_AUTH_JWT_ISSUER)
+        # FIXME: The env below is triggerring: did not find main branch: main,...
+        # self.with_env("OPAL_STATISTICS_ENABLED", s.OPAL_STATISTICS_ENABLED)
+
+
+class OpalClientContainer(DockerContainer):
+    def __init__(
+        self,
+        image: str = f"permitio/opal-client:{s.OPAL_IMAGE_TAG}",
+        docker_client_kw: dict | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(image, docker_client_kw, **kwargs)
+
+        self.with_name(s.OPAL_TESTS_CLIENT_CONTAINER_NAME)
+        self.with_exposed_ports(7000, 8181)
+
+        if s.OPAL_TESTS_DEBUG:
+            self.with_env("LOG_DIAGNOSE", "true")
+            self.with_env("OPAL_LOG_LEVEL", "DEBUG")
+
+        self.with_env("OPAL_LOG_FORMAT_INCLUDE_PID", s.OPAL_LOG_FORMAT_INCLUDE_PID)
+        self.with_env("OPAL_INLINE_OPA_LOG_FORMAT", s.OPAL_INLINE_OPA_LOG_FORMAT)
+        self.with_env(
+            "OPAL_SHOULD_REPORT_ON_DATA_UPDATES", s.OPAL_SHOULD_REPORT_ON_DATA_UPDATES
+        )
+        self.with_env("OPAL_DEFAULT_UPDATE_CALLBACKS", s.OPAL_DEFAULT_UPDATE_CALLBACKS)
+        self.with_env(
+            "OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED",
+            s.OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED,
+        )
+        self.with_env("OPAL_CLIENT_TOKEN", s.OPAL_CLIENT_TOKEN)
+        self.with_env("OPAL_AUTH_PUBLIC_KEY", s.OPAL_AUTH_PUBLIC_KEY)
+        self.with_env("OPAL_AUTH_JWT_AUDIENCE", s.OPAL_AUTH_JWT_AUDIENCE)
+        self.with_env("OPAL_AUTH_JWT_ISSUER", s.OPAL_AUTH_JWT_ISSUER)
+        # self.with_env("OPAL_STATISTICS_ENABLED", s.OPAL_STATISTICS_ENABLED)

tests/run.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+if [[ -f ".env" ]]; then
+  # shellcheck disable=SC1091
+  source .env
+fi
+
+# TODO: Disable after debugging.
+export OPAL_TESTS_DEBUG='true'
+export OPAL_POLICY_REPO_URL
+export OPAL_POLICY_REPO_BRANCH
+export OPAL_POLICY_REPO_SSH_KEY
+export OPAL_AUTH_PUBLIC_KEY
+export OPAL_AUTH_PRIVATE_KEY
+
+OPAL_POLICY_REPO_URL=${OPAL_POLICY_REPO_URL:[email protected]:permitio/opal-tests-policy-repo.git}
+OPAL_POLICY_REPO_BRANCH=test-$RANDOM$RANDOM
+OPAL_POLICY_REPO_SSH_KEY_PATH=${OPAL_POLICY_REPO_SSH_KEY_PATH:-~/.ssh/id_rsa}
+OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY:-$(cat "$OPAL_POLICY_REPO_SSH_KEY_PATH")}
+
+function generate_opal_keys {
+  echo "- Generating OPAL keys"
+
+  ssh-keygen -q -t rsa -b 4096 -m pem -f opal_crypto_key -N ""
+  OPAL_AUTH_PUBLIC_KEY="$(cat opal_crypto_key.pub)"
+  OPAL_AUTH_PRIVATE_KEY="$(tr '\n' '_' <opal_crypto_key)"
+  rm opal_crypto_key.pub opal_crypto_key
+}
+
+function main {
+  # Setup
+  generate_opal_keys
+
+  pytest -s
+}
+
+main

tests/settings.py

@@ -0,0 +1,135 @@
+import io
+import json
+
+from contextlib import redirect_stdout
+from os import getenv as _
+from secrets import token_hex
+
+from opal_common.cli.commands import obtain_token
+from opal_common.schemas.security import PeerType
+from testcontainers.core.generic import DockerContainer
+from testcontainers.core.waiting_utils import wait_for_logs
+
+OPAL_TESTS_DEBUG = _("OPAL_TESTS_DEBUG") is not None
+
+OPAL_TESTS_UNIQ_ID = token_hex(2)
+OPAL_TESTS_NETWORK_NAME = f"pytest_opal_{OPAL_TESTS_UNIQ_ID}"
+OPAL_TESTS_SERVER_CONTAINER_NAME = f"pytest_opal_server_{OPAL_TESTS_UNIQ_ID}"
+OPAL_TESTS_CLIENT_CONTAINER_NAME = f"pytest_opal_client_{OPAL_TESTS_UNIQ_ID}"
+
+OPAL_IMAGE_TAG = _("OPAL_IMAGE_TAG", "latest")
+
+OPAL_AUTH_PUBLIC_KEY = _("OPAL_AUTH_PUBLIC_KEY", "")
+OPAL_AUTH_PRIVATE_KEY = _("OPAL_AUTH_PRIVATE_KEY", "")
+OPAL_AUTH_PRIVATE_KEY_PASSPHRASE = _("OPAL_AUTH_PRIVATE_KEY_PASSPHRASE")
+OPAL_AUTH_MASTER_TOKEN = _("OPAL_AUTH_MASTER_TOKEN", token_hex(16))
+OPAL_AUTH_JWT_AUDIENCE = _("OPAL_AUTH_JWT_AUDIENCE", "https://api.opal.ac/v1/")
+OPAL_AUTH_JWT_ISSUER = _("OPAL_AUTH_JWT_ISSUER", "https://opal.ac/")
+
+# Temporary container to generate the required tokens.
+_container = (
+    DockerContainer(f"permitio/opal-server:{OPAL_IMAGE_TAG}")
+    .with_exposed_ports(7002)
+    .with_env("OPAL_REPO_WATCHER_ENABLED", "0")
+    .with_env("OPAL_AUTH_PUBLIC_KEY", OPAL_AUTH_PUBLIC_KEY)
+    .with_env("OPAL_AUTH_PRIVATE_KEY", OPAL_AUTH_PRIVATE_KEY)
+    .with_env("OPAL_AUTH_MASTER_TOKEN", OPAL_AUTH_MASTER_TOKEN)
+    .with_env("OPAL_AUTH_JWT_AUDIENCE", OPAL_AUTH_JWT_AUDIENCE)
+    .with_env("OPAL_AUTH_JWT_ISSUER", OPAL_AUTH_JWT_ISSUER)
+)
+
+OPAL_CLIENT_TOKEN = _("OPAL_CLIENT_TOKEN")
+OPAL_DATA_SOURCE_TOKEN = _("OPAL_DATA_SOURCE_TOKEN")
+
+with _container:
+    wait_for_logs(_container, "OPAL Server Startup")
+    kwargs = {
+        "master_token": OPAL_AUTH_MASTER_TOKEN,
+        "server_url": f"http://{_container.get_container_host_ip()}:{_container.get_exposed_port(7002)}",
+        "ttl": (365, "days"),
+        "claims": {},
+    }
+
+    if not OPAL_CLIENT_TOKEN:
+        with io.StringIO() as stdout:
+            with redirect_stdout(stdout):
+                obtain_token(type=PeerType("client"), **kwargs)
+            OPAL_CLIENT_TOKEN = stdout.getvalue().strip()
+
+    if not OPAL_DATA_SOURCE_TOKEN:
+        with io.StringIO() as stdout:
+            with redirect_stdout(stdout):
+                obtain_token(type=PeerType("datasource"), **kwargs)
+            OPAL_DATA_SOURCE_TOKEN = stdout.getvalue().strip()
+
+UVICORN_NUM_WORKERS = _("UVICORN_NUM_WORKERS", "4")
+OPAL_STATISTICS_ENABLED = _("OPAL_STATISTICS_ENABLED", "true")
+
+OPAL_POLICY_REPO_URL = _(
+    "OPAL_POLICY_REPO_URL", "[email protected]:permitio/opal-tests-policy-repo.git"
+)
+OPAL_POLICY_REPO_SSH_KEY = _("OPAL_POLICY_REPO_SSH_KEY", "")
+OPAL_POLICY_REPO_MAIN_BRANCH = _("POLICY_REPO_BRANCH", "main")
+OPAL_POLICY_REPO_POLLING_INTERVAL = _("OPAL_POLICY_REPO_POLLING_INTERVAL", "30")
+OPAL_LOG_FORMAT_INCLUDE_PID = _("OPAL_LOG_FORMAT_INCLUDE_PID ", "true")
+OPAL_POLICY_REPO_WEBHOOK_SECRET = _("OPAL_POLICY_REPO_WEBHOOK_SECRET", "xxxxx")
+OPAL_POLICY_REPO_WEBHOOK_PARAMS = _(
+    "OPAL_POLICY_REPO_WEBHOOK_PARAMS",
+    json.dumps(
+        {
+            "secret_header_name": "x-webhook-token",
+            "secret_type": "token",
+            "secret_parsing_regex": "(.*)",
+            "event_request_key": "gitEvent",
+            "push_event_value": "git.push",
+        }
+    ),
+)
+
+_url = f"http://{OPAL_TESTS_SERVER_CONTAINER_NAME}.{OPAL_TESTS_NETWORK_NAME}:7002/policy-data"
+OPAL_DATA_CONFIG_SOURCES = json.dumps(
+    {
+        "config": {
+            "entries": [
+                {
+                    "url": _url,
+                    "config": {
+                        "headers": {"Authorization": f"Bearer {OPAL_CLIENT_TOKEN}"}
+                    },
+                    "topics": ["policy_data"],
+                    "dst_path": "/static",
+                }
+            ]
+        }
+    }
+)
+
+# Opal Client
+OPAL_INLINE_OPA_LOG_FORMAT = "http"
+OPAL_SHOULD_REPORT_ON_DATA_UPDATES = "true"
+OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED = "true"
+OPAL_DEFAULT_UPDATE_CALLBACKS = json.dumps(
+    {
+        "callbacks": [
+            [
+                f"http://{OPAL_TESTS_SERVER_CONTAINER_NAME}.{OPAL_TESTS_NETWORK_NAME}:7002/data/callback_report",
+                {
+                    "method": "post",
+                    "process_data": False,
+                    "headers": {
+                        "Authorization": f"Bearer {OPAL_CLIENT_TOKEN}",
+                        "content-type": "application/json",
+                    },
+                },
+            ]
+        ]
+    }
+)
+
+
+def dump_settings():
+    with open(f"pytest_{OPAL_TESTS_UNIQ_ID}.env", "w") as envfile:
+        envfile.write("#!/usr/bin/env bash\n\n")
+        for key, val in globals().items():
+            if key.startswith("OPAL") or key.startswith("UVICORN"):
+                envfile.write(f"export {key}='{val}'\n\n")

tests/test_app.py

@@ -0,0 +1,3 @@
+# TODO: Replace once all fixtures are properly working.
+def test_trivial():
+    assert 4 + 1 == 5