|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | +package org.apache.pulsar.tests.integration.containers; |
| 20 | + |
| 21 | +import static java.nio.charset.StandardCharsets.UTF_8; |
| 22 | +import static org.assertj.core.api.Assertions.assertThat; |
| 23 | +import io.netty.buffer.ByteBufAllocator; |
| 24 | +import java.nio.file.Files; |
| 25 | +import java.nio.file.Path; |
| 26 | +import java.time.Duration; |
| 27 | +import java.util.UUID; |
| 28 | +import java.util.concurrent.CountDownLatch; |
| 29 | +import java.util.zip.GZIPInputStream; |
| 30 | +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; |
| 31 | +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; |
| 32 | +import org.apache.commons.io.FileUtils; |
| 33 | +import org.apache.pulsar.tests.ExtendedNettyLeakDetector; |
| 34 | +import org.testcontainers.containers.wait.strategy.Wait; |
| 35 | +import org.testcontainers.images.builder.Transferable; |
| 36 | +import org.testcontainers.utility.MountableFile; |
| 37 | +import org.testng.SkipException; |
| 38 | +import org.testng.annotations.DataProvider; |
| 39 | +import org.testng.annotations.Test; |
| 40 | + |
| 41 | +public class NettyLeakDetectionTest { |
| 42 | + @DataProvider |
| 43 | + public Object[][] shutdownModes() { |
| 44 | + return new Object[][] {{false}, {true}}; |
| 45 | + } |
| 46 | + |
| 47 | + @Test(dataProvider = "shutdownModes") |
| 48 | + public void collectsLeaksReportedDuringShutdown(boolean supervised) throws Exception { |
| 49 | + if (!ExtendedNettyLeakDetector.isExtendedNettyLeakDetectorEnabled() |
| 50 | + || !"paranoid".equals(System.getProperty("io.netty.leakDetection.level"))) { |
| 51 | + throw new SkipException("Requires the default paranoid test leak detector"); |
| 52 | + } |
| 53 | + var container = new LeakProbeContainer(supervised); |
| 54 | + Path logs = Path.of(System.getProperty("buildDirectory", "build"), |
| 55 | + "container-logs", container.getContainerName()); |
| 56 | + try (container) { |
| 57 | + container.start(); |
| 58 | + assertThat(container.execCmd("sh", "-c", "ls /var/log/pulsar/netty_leak_*.txt 2>/dev/null || true") |
| 59 | + .getStdout()).as("No leak report before JVM shutdown").isEmpty(); |
| 60 | + container.stop(); |
| 61 | + Path archive = logs.resolve("var-log-pulsar.tar.gz"); |
| 62 | + boolean foundLeak = false; |
| 63 | + try (var tar = new TarArchiveInputStream(new GZIPInputStream(Files.newInputStream(archive)))) { |
| 64 | + TarArchiveEntry entry; |
| 65 | + while ((entry = tar.getNextEntry()) != null) { |
| 66 | + if (entry.isFile() && entry.getName().contains("netty_leak_")) { |
| 67 | + String report = new String(tar.readAllBytes(), UTF_8); |
| 68 | + assertThat(report).contains("Traced leak detected ByteBuf", "container-shutdown-leak"); |
| 69 | + foundLeak = true; |
| 70 | + } |
| 71 | + } |
| 72 | + } |
| 73 | + assertThat(foundLeak).as("Shutdown leak included in the collected container logs").isTrue(); |
| 74 | + } finally { |
| 75 | + // This test deliberately leaks in a separate JVM. Do not report its expected leak in CI. |
| 76 | + FileUtils.deleteDirectory(logs.toFile()); |
| 77 | + } |
| 78 | + } |
| 79 | + |
| 80 | + private static class LeakProbeContainer extends PulsarContainer<LeakProbeContainer> { |
| 81 | + LeakProbeContainer(boolean supervised) { |
| 82 | + super("leak-test-" + UUID.randomUUID(), "probe", "probe", |
| 83 | + supervised ? "/usr/bin/supervisord" : "bin/pulsar", INVALID_PORT, INVALID_PORT); |
| 84 | + String className = LeakProbe.class.getName(); |
| 85 | + String resource = className.replace('.', '/') + ".class"; |
| 86 | + withCopyFileToContainer(MountableFile.forClasspathResource(resource), "/tmp/" + resource); |
| 87 | + String script = "#!/bin/sh\nexec java $PULSAR_EXTRA_OPTS -cp '/pulsar/lib/*:/tmp' '" |
| 88 | + + className + "'\n"; |
| 89 | + if (supervised) { |
| 90 | + withCopyToContainer(Transferable.of(script, 0755), "/tmp/leak-probe.sh"); |
| 91 | + withCopyToContainer(Transferable.of(""" |
| 92 | + [program:leak-probe] |
| 93 | + command=/tmp/leak-probe.sh |
| 94 | + autostart=true |
| 95 | + autorestart=false |
| 96 | + stopwaitsecs=15 |
| 97 | + """), "/etc/supervisord/conf.d/leak-probe.conf"); |
| 98 | + withCommand("-c", "/etc/supervisord.conf"); |
| 99 | + } else { |
| 100 | + // Use the standalone shutdown path with a small JVM instead of starting a broker. |
| 101 | + withCopyToContainer(Transferable.of(script, 0755), "/pulsar/bin/pulsar"); |
| 102 | + withCommand(); |
| 103 | + } |
| 104 | + waitingFor(Wait.forSuccessfulCommand("test -f /tmp/leak-probe-ready") |
| 105 | + .withStartupTimeout(Duration.ofSeconds(60))); |
| 106 | + } |
| 107 | + |
| 108 | + @Override |
| 109 | + protected void passNettyLeakDetectionSystemProperties() { |
| 110 | + super.passNettyLeakDetectionSystemProperties(); |
| 111 | + // Keep the deliberate leak alive until shutdown even when local tests fail on leaks. |
| 112 | + appendToEnv("PULSAR_EXTRA_OPTS", |
| 113 | + "-D" + ExtendedNettyLeakDetector.EXIT_JVM_ON_LEAK_SYSTEM_PROPERTY_NAME + "=false"); |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + public static class LeakProbe { |
| 118 | + public static void main(String[] args) throws Exception { |
| 119 | + ExtendedNettyLeakDetector.setInitialHint("container-shutdown-leak"); |
| 120 | + leakBuffer(); |
| 121 | + Files.writeString(Path.of("/tmp/leak-probe-ready"), "ready"); |
| 122 | + // Only the detector's shutdown hook will force collection and report the leaked buffer. |
| 123 | + new CountDownLatch(1).await(); |
| 124 | + } |
| 125 | + |
| 126 | + private static void leakBuffer() { |
| 127 | + ByteBufAllocator.DEFAULT.directBuffer(16).writeLong(42); |
| 128 | + } |
| 129 | + } |
| 130 | +} |
0 commit comments