Add zstd to native access (#105715)

This commit makes zstd compression available to Elasticsearch. The
library is pulled in through maven in jar files for each platform, then
bundled in a new platform directory under lib. Access to the zstd
compression/decompression is through NativeAccess.

Author: rjernst

build-tools-internal/build.gradle

@@ -119,6 +119,10 @@ gradlePlugin {
       id = 'elasticsearch.java-doc'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavadocPlugin'
     }
+    javaBase {
+      id = 'elasticsearch.java-base'
+      implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin'
+    }
     java {
       id = 'elasticsearch.java'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal
 
-import spock.lang.TempDir
+
 import spock.lang.Unroll
 import com.github.tomakehurst.wiremock.WireMockServer
 
@@ -103,10 +103,6 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
             plugins {
              id 'elasticsearch.jdk-download' apply false
             }
-
-            subprojects {
-
-            }
         """
         3.times {
             subProject(':sub-' + it) << """

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -9,6 +9,7 @@
 import org.elasticsearch.gradle.util.Pair
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.TestUtil
 import org.jetbrains.gradle.ext.JUnit
 
 import java.nio.file.Files
@@ -128,9 +129,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',
       ':server:generatePluginsList',
-      ':generateProviderImpls'].collect { elasticsearchProject.right()?.task(it) ?: it })
+      ':generateProviderImpls',
+      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 
+  // this path is produced by the extractLibs task above
+  String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
+
   idea {
     project {
       vcs = 'Git'
@@ -162,6 +167,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.library.path=' + testLibraryPath,
+              '-Djna.library.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -12,11 +12,15 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.test.TestUtil;
+import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolutionStrategy;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
@@ -26,10 +30,13 @@
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.GroovyCompile;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
 
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
 import javax.inject.Inject;
 
@@ -59,6 +66,7 @@ public void apply(Project project) {
         configureConfigurations(project);
         configureCompile(project);
         configureInputNormalization(project);
+        configureNativeLibraryPath(project);
 
         // convenience access to common versions used in dependencies
         project.getExtensions().getExtraProperties().set("versions", VersionProperties.getVersions());
@@ -165,6 +173,26 @@ public static void configureInputNormalization(Project project) {
         project.getNormalization().getRuntimeClasspath().ignore("IMPL-JARS/**/META-INF/MANIFEST.MF");
     }
 
+    private static void configureNativeLibraryPath(Project project) {
+        String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries";
+        Configuration nativeConfig = project.getConfigurations().create("nativeLibs");
+        nativeConfig.defaultDependencies(deps -> {
+            deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default")));
+        });
+        // This input to the following lambda needs to be serializable. Configuration is not serializable, but FileCollection is.
+        FileCollection nativeConfigFiles = nativeConfig;
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            var systemProperties = test.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class);
+            var libraryPath = (Supplier<String>) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath());
+
+            test.dependsOn(nativeConfigFiles);
+            // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops
+            systemProperties.systemProperty("java.library.path", libraryPath);
+            systemProperties.systemProperty("jna.library.path", libraryPath);
+        });
+    }
+
     private static Provider<Integer> releaseVersionProviderFromCompileTask(Project project, AbstractCompile compileTask) {
         return project.provider(() -> {
             JavaVersion javaVersion = JavaVersion.toVersion(compileTask.getTargetCompatibility());

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -105,7 +105,7 @@ private void addMrjarSourceset(
             testTask.dependsOn(jarTask);
 
             SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
-            FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
+            FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput();
             FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
             testTask.setClasspath(testRuntime.minus(mainRuntime).plus(project.files(jarTask)));
         });

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.ElasticsearchDistribution;
+
+import java.util.Locale;
+
+public class TestUtil {
+
+    public static String getTestLibraryPath(String nativeLibsDir) {
+        String arch = Architecture.current().toString().toLowerCase(Locale.ROOT);
+        String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch);
+        String existingLibraryPath = System.getProperty("java.library.path");
+
+        return String.format(Locale.ROOT, "%s/%s:%s", nativeLibsDir, platform, existingLibraryPath);
+    }
+}

build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -50,6 +50,13 @@ abstract class AbstractGradleFuncTest extends Specification {
         propertiesFile = testProjectDir.newFile('gradle.properties')
         propertiesFile <<
             "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
+
+        def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries")
+        nativeLibsProject << """
+            plugins {
+                id 'base'
+            }
+        """
     }
 
     def cleanup() {

distribution/archives/build.gradle

@@ -15,7 +15,7 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles(os, architecture)
       }
       into('config') {
         dirMode 0750

distribution/build.gradle

@@ -261,7 +261,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,14 +292,15 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
+    libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries')
   }
 
   project.ext {
 
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles =
+    libFiles = { os, architecture ->
       copySpec {
         // Delay by using closures, since they have not yet been configured, so no jar task exists yet.
         from(configurations.libs)
@@ -330,7 +331,14 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         into('tools/ansi-console') {
           from(configurations.libsAnsiConsole)
         }
+        into('platform') {
+          from(configurations.libsNative)
+          if (os != null) {
+            include (os + '-' + architecture + '/*')
+          }
+        }
       }
+    }
 
     modulesFiles = { os, architecture ->
       copySpec {

distribution/packages/build.gradle

@@ -131,6 +131,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // top level "into" directive is not inherited from ospackage for some reason, so we must
     // specify it again explicitly for copying common files
+    String platform = 'linux-' + ((architecture == 'x64') ? 'x86_64' : architecture)
     into('/usr/share/elasticsearch') {
       into('bin') {
         with binFiles(type, false)
@@ -140,7 +141,7 @@ def commonPackageConfig(String type, String architecture) {
         fileMode 0644
       }
       into('lib') {
-        with libFiles
+        with libFiles('linux', architecture)
       }
       into('modules') {
         with modulesFiles('linux', architecture)

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -10,7 +10,11 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.SuppressForbidden;
 
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -21,6 +25,8 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
+        String libraryPath = findLibraryPath(sysprops);
+
         return Stream.of(
             /*
              * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl;
@@ -71,6 +77,8 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeOverrideDockerCgroup(distroType),
             maybeSetActiveProcessorCount(nodeSettings),
             setReplayFile(distroType, isHotspot),
+            "-Djava.library.path=" + libraryPath,
+            "-Djna.library.path=" + libraryPath,
             // Pass through distribution type
             "-Des.distribution.type=" + distroType
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
@@ -127,4 +135,38 @@ private static String maybeEnableNativeAccess() {
         }
         return "";
     }
+
+    private static String findLibraryPath(Map<String, String> sysprops) {
+        // working dir is ES installation, so we use relative path here
+        Path platformDir = Paths.get("lib", "platform");
+        String existingPath = sysprops.get("java.library.path");
+        assert existingPath != null;
+
+        String osname = sysprops.get("os.name");
+        String os;
+        if (osname.startsWith("Windows")) {
+            os = "windows";
+        } else if (osname.startsWith("Linux")) {
+            os = "linux";
+        } else if (osname.startsWith("Mac OS")) {
+            os = "darwin";
+        } else {
+            os = "unsupported_os[" + osname + "]";
+        }
+        String archname = sysprops.get("os.arch");
+        String arch;
+        if (archname.equals("amd64")) {
+            arch = "x64";
+        } else if (archname.equals("aarch64")) {
+            arch = archname;
+        } else {
+            arch = "unsupported_arch[" + archname + "]";
+        }
+        return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath;
+    }
+
+    @SuppressForbidden(reason = "no way to get path separator with nio")
+    private static String getPathSeparator() {
+        return File.pathSeparator;
+    }
 }