feat(CI): Set up CodSpeed

.github/workflows/codspeed.yml

@@ -0,0 +1,42 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read  # Check out the repository and its submodules
+  id-token: write  # Authenticate result uploads to CodSpeed via OIDC
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          submodules: recursive
+          persist-credentials: false
+
+      - name: Set up JDK
+        uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad  # v6.1.0
+        with:
+          distribution: temurin
+          java-version: 21
+          cache: maven
+
+      - name: Set up Maven
+        uses: stCarolas/setup-maven@12eb41b233df95d49b0c11fc1b5bc8312e5d4ce0  # v5.1
+        with:
+          maven-version: 3.10.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062  # v5.6.0
+        with:
+          mode: walltime
+          run: make benchmark

.gitmodules

@@ -1,4 +1,11 @@
 [submodule "src/test/java/com/flagsmith/flagengine/enginetestdata"]
 	path = src/test/java/com/flagsmith/flagengine/enginetestdata
 	url = git@github.com:Flagsmith/engine-test-data.git
-	branch = v3.7.0
+	branch = v3.7.0
+[submodule "third-party/codspeed-jvm"]
+	path = third-party/codspeed-jvm
+	url = https://github.com/CodSpeedHQ/codspeed-jvm.git
+[submodule "third-party/sdk-benchmarks"]
+	path = third-party/sdk-benchmarks
+	url = https://github.com/Flagsmith/sdk-benchmarks.git
+	branch = 0.0.1

Makefile

@@ -0,0 +1,18 @@
+BENCHMARKS_JAR ?= target/benchmarks.jar
+CODSPEED_JVM := third-party/codspeed-jvm
+SDK_BENCHMARKS := third-party/sdk-benchmarks
+
+$(CODSPEED_JVM)/gradlew:
+	git submodule update --init -- $(CODSPEED_JVM)
+	git -C $(CODSPEED_JVM) submodule update --init -- jmh-fork/jmh-core/native-instrument-hooks/instrument-hooks
+
+$(SDK_BENCHMARKS)/environments:
+	git submodule update --init -- $(SDK_BENCHMARKS)
+
+$(BENCHMARKS_JAR): pom.xml $(shell find src/main src/jmh -type f) $(wildcard $(SDK_BENCHMARKS)/environments/*.json) | $(CODSPEED_JVM)/gradlew $(SDK_BENCHMARKS)/environments
+	cd third-party/codspeed-jvm && ./gradlew -p jmh-fork publishToMavenLocal
+	mvn --batch-mode --no-transfer-progress -P benchmarks -DskipTests package
+
+.PHONY: benchmark
+benchmark: $(BENCHMARKS_JAR)
+	java -jar $(BENCHMARKS_JAR)

pom.xml

@@ -301,6 +301,10 @@
           <failsOnError>true</failsOnError>
           <linkXRef>true</linkXRef>
           <violationSeverity>warning</violationSeverity>
+          <sourceDirectories>
+            <sourceDirectory>${project.build.sourceDirectory}</sourceDirectory>
+            <sourceDirectory>${project.basedir}/src/jmh/java</sourceDirectory>
+          </sourceDirectories>
         </configuration>
         <executions>
           <execution>
@@ -388,5 +392,97 @@
         <okhttp.mockwebserver.version>4.12.0</okhttp.mockwebserver.version>
       </properties>
     </profile>
+    <profile>
+      <!-- CodSpeed JMH benchmarks (JDK 21+), see https://codspeed.io/docs/benchmarks/java -->
+      <id>benchmarks</id>
+      <properties>
+        <codspeed.jmh.version>0.2.1</codspeed.jmh.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>io.codspeed.jmh</groupId>
+          <artifactId>jmh-core</artifactId>
+          <version>${codspeed.jmh.version}</version>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <version>3.6.2</version>
+            <executions>
+              <execution>
+                <id>add-jmh-sources</id>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>add-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>${project.basedir}/src/jmh/java</source>
+                  </sources>
+                </configuration>
+              </execution>
+              <execution>
+                <id>add-jmh-resources</id>
+                <phase>generate-resources</phase>
+                <goals>
+                  <goal>add-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>${project.basedir}/third-party/sdk-benchmarks</directory>
+                      <includes>
+                        <include>environments/*.json</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-compiler-plugin</artifactId>
+            <configuration>
+              <source>21</source>
+              <target>21</target>
+              <annotationProcessorPaths combine.children="append">
+                <path>
+                  <groupId>io.codspeed.jmh</groupId>
+                  <artifactId>jmh-generator-annprocess</artifactId>
+                  <version>${codspeed.jmh.version}</version>
+                </path>
+              </annotationProcessorPaths>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>3.6.2</version>
+            <executions>
+              <execution>
+                <phase>package</phase>
+                <goals>
+                  <goal>shade</goal>
+                </goals>
+                <configuration>
+                  <outputFile>${project.build.directory}/benchmarks.jar</outputFile>
+                  <createDependencyReducedPom>false</createDependencyReducedPom>
+                  <transformers>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                      <mainClass>org.openjdk.jmh.Main</mainClass>
+                    </transformer>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                  </transformers>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>

src/jmh/java/com/flagsmith/SdkBenchmark.java

@@ -0,0 +1,124 @@
+package com.flagsmith;
+
+import com.flagsmith.config.FlagsmithConfig;
+import com.flagsmith.exceptions.FlagsmithClientError;
+import com.flagsmith.flagengine.EvaluationContext;
+import com.flagsmith.models.Segment;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+import okhttp3.Interceptor;
+import okhttp3.MediaType;
+import okhttp3.Protocol;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+/**
+ * Benchmarks a locally evaluating SDK client against environment documents tiered by size.
+ */
+@State(Scope.Benchmark)
+public class SdkBenchmark {
+  private static final MediaType JSON = MediaType.get("application/json");
+  private static final String IDENTITY_OVERRIDES_IDENTIFIER = "identity-override-1";
+  private static final String IDENTIFIER = "user-1";
+  private static final Map<String, Object> TRAITS = Map.of(
+      "plan", "plan-7",
+      "age", 30,
+      "country", "GB",
+      "version", "2.5.1",
+      "email", "jane@example.com",
+      "user_id", 12,
+      "beta", true,
+      "score", 95,
+      "tier", "pro",
+      "device", "iPhone 15");
+
+  @Param({"small", "medium", "large"})
+  public String environment;
+
+  private FlagsmithClient client;
+
+  /**
+   * Build a locally evaluating client serving the environment document from memory.
+   */
+  @Setup
+  public void buildClient() throws IOException {
+    byte[] environmentDocument;
+    try (InputStream stream = getClass().getResourceAsStream(
+        "/environments/" + environment + ".json")) {
+      environmentDocument = stream.readAllBytes();
+    }
+
+    client = FlagsmithClient.newBuilder()
+        .setApiKey("ser.benchmark")
+        .withConfiguration(FlagsmithConfig.newBuilder()
+            .baseUri("http://flagsmith.benchmark/api/v1/")
+            .addHttpInterceptor(serveEnvironmentDocument(environmentDocument))
+            .withLocalEvaluation(true)
+            .withEnvironmentRefreshIntervalSeconds(86400) // Refreshed by benchmarks only
+            .build())
+        .build();
+  }
+
+  @TearDown
+  public void closeClient() {
+    client.close();
+  }
+
+  /**
+   * Get a feature value from the environment flags.
+   */
+  @Benchmark
+  public Object getEnvironmentFlags() throws FlagsmithClientError {
+    return client.getEnvironmentFlags().getFeatureValue("feature_1");
+  }
+
+  /**
+   * Check a feature for an identity with identity overrides.
+   */
+  @Benchmark
+  public boolean getIdentityFlagsWithIdentityOverrides() throws FlagsmithClientError {
+    return client.getIdentityFlags(IDENTITY_OVERRIDES_IDENTIFIER).isFeatureEnabled("feature_8");
+  }
+
+  /**
+   * Get a feature value for an identity without traits.
+   */
+  @Benchmark
+  public Object getIdentityFlagsNoTraits() throws FlagsmithClientError {
+    return client.getIdentityFlags(IDENTIFIER).getFeatureValue("feature_1");
+  }
+
+  /**
+   * Get a feature value for an identity with traits.
+   */
+  @Benchmark
+  public Object getIdentityFlagsWithTraits() throws FlagsmithClientError {
+    return client.getIdentityFlags(IDENTIFIER, TRAITS).getFeatureValue("feature_1");
+  }
+
+  /**
+   * List the segments of an identity with traits.
+   */
+  @Benchmark
+  public List<Segment> getIdentitySegmentsWithTraits() throws FlagsmithClientError {
+    return client.getIdentitySegments(IDENTIFIER, TRAITS);
+  }
+
+  private static Interceptor serveEnvironmentDocument(byte[] environmentDocument) {
+    return chain -> new Response.Builder()
+        .request(chain.request())
+        .protocol(Protocol.HTTP_1_1)
+        .code(200)
+        .message("OK")
+        .body(ResponseBody.create(environmentDocument, JSON))
+        .build();
+  }
+}