PowerFlex/ScaleIO SDC client connection improvements (#9268)

* Mitigation for non-scalable Powerflex/ScaleIO clients
- Added ScaleIOSDCManager to manage SDC connections, checks clients limit, prepare and unprepare SDC on the hosts.
- Added commands for prepare and unprepare storage clients to prepare/start and stop SDC service respectively on the hosts.
- Introduced config 'storage.pool.connected.clients.limit' at storage level for client limits, currently support for Powerflex only.

* tests issue fixed

* refactor / improvements

* lock with powerflex systemid while checking connections limit

* updated powerflex systemid lock to hold till sdc preparation

* Added custom stats support for storage pool, through listStoragePools API

* code improvements, and unit tests

* unit tests fixes

* Update config 'storage.pool.connected.clients.limit' to dynamic, and some improvements

* Stop SDC on host after migration if no volumes mapped to host

* Wait for SDC to connect after scini service start, and some log improvements

* Do not throw exception (log it) when SDC is not connected while revoking access for the powerflex volume

* some log improvements

Author: sureshanaparti

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -439,6 +439,7 @@ public class ApiConstants {
     public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
     public static final String STORAGE_CAPABILITIES = "storagecapabilities";
+    public static final String STORAGE_CUSTOM_STATS = "storagecustomstats";
     public static final String SUBNET = "subnet";
     public static final String OWNER = "owner";
     public static final String SWAP_OWNER = "swapowner";

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java

@@ -74,7 +74,8 @@ public class ListStoragePoolsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "host ID of the storage pools")
     private Long hostId;
 
-
+    @Parameter(name = ApiConstants.STORAGE_CUSTOM_STATS, type = CommandType.BOOLEAN, description = "If true, lists the custom stats of the storage pool", since = "4.18.1")
+    private Boolean customStats;
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -131,6 +132,10 @@ public void setScope(String scope) {
         this.scope = scope;
     }
 
+    public Boolean getCustomStats() {
+        return customStats != null && customStats;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -97,6 +97,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "total min IOPS currently in use by volumes")
     private Long allocatedIops;
 
+    @SerializedName(ApiConstants.STORAGE_CUSTOM_STATS)
+    @Param(description = "the storage pool custom stats", since = "4.18.1")
+    private Map<String, String> customStats;
+
     @SerializedName("tags")
     @Param(description = "the tags for the storage pool")
     private String tags;
@@ -304,6 +308,14 @@ public void setAllocatedIops(Long allocatedIops) {
        this.allocatedIops = allocatedIops;
     }
 
+    public Map<String, String> getCustomStats() {
+        return customStats;
+    }
+
+    public void setCustomStats(Map<String, String> customStats) {
+        this.customStats = customStats;
+    }
+
     public String getTags() {
         return tags;
     }

core/src/main/java/com/cloud/agent/api/PrepareStorageClientAnswer.java

@@ -0,0 +1,43 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import java.util.Map;
+
+public class PrepareStorageClientAnswer extends Answer {
+    Map<String, String> detailsMap;
+
+    public PrepareStorageClientAnswer() {
+        super();
+    }
+
+    public PrepareStorageClientAnswer(Command command, boolean success, Map<String, String> detailsMap) {
+        super(command, success, "");
+        this.detailsMap = detailsMap;
+    }
+
+    public PrepareStorageClientAnswer(Command command, boolean success, String details) {
+        super(command, success, details);
+    }
+
+    public Map<String, String> getDetailsMap() {
+        return detailsMap;
+    }
+}

core/src/main/java/com/cloud/agent/api/PrepareStorageClientCommand.java

@@ -0,0 +1,56 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import java.util.Map;
+
+import com.cloud.storage.Storage.StoragePoolType;
+
+public class PrepareStorageClientCommand extends Command {
+    private StoragePoolType poolType;
+    private String poolUuid;
+    private Map<String, String> details;
+
+    public PrepareStorageClientCommand() {
+    }
+
+    public PrepareStorageClientCommand(StoragePoolType poolType, String poolUuid, Map<String, String> details) {
+        this.poolType = poolType;
+        this.poolUuid = poolUuid;
+        this.details = details;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+
+    public Map<String, String> getDetails() {
+        return details;
+    }
+}

core/src/main/java/com/cloud/agent/api/UnprepareStorageClientAnswer.java

@@ -0,0 +1,34 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+public class UnprepareStorageClientAnswer extends Answer {
+    public UnprepareStorageClientAnswer() {
+        super();
+    }
+
+    public UnprepareStorageClientAnswer(Command command, boolean success) {
+        super(command, success, "");
+    }
+
+    public UnprepareStorageClientAnswer(Command command, boolean success, String details) {
+        super(command, success, details);
+    }
+}

core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java

@@ -0,0 +1,48 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.storage.Storage.StoragePoolType;
+
+public class UnprepareStorageClientCommand extends Command {
+    private StoragePoolType poolType;
+    private String poolUuid;
+
+    public UnprepareStorageClientCommand() {
+    }
+
+    public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid) {
+        this.poolType = poolType;
+        this.poolUuid = poolUuid;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+}

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java

@@ -18,6 +18,8 @@
  */
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
+import java.util.Map;
+
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.storage.command.CommandResult;
 
@@ -86,6 +88,22 @@ default boolean requiresAccessForMigration(DataObject dataObject) {
      */
     boolean canProvideStorageStats();
 
+    /**
+     * intended for managed storage
+     * returns true if the storage can provide its custom stats
+     */
+    default boolean poolProvidesCustomStorageStats() {
+        return false;
+    }
+
+    /**
+     * intended for managed storage
+     * returns the custom stats if the storage can provide them
+     */
+    default Map<String, String> getCustomStorageStats(StoragePool pool) {
+        return null;
+    }
+
     /**
      * intended for managed storage
      * returns the total capacity and used size in bytes
@@ -110,6 +128,14 @@ default boolean requiresAccessForMigration(DataObject dataObject) {
      */
     boolean canHostAccessStoragePool(Host host, StoragePool pool);
 
+    /**
+     * intended for managed storage
+     * returns true if the host can prepare storage client to provide access the storage pool
+     */
+    default boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool) {
+        return false;
+    }
+
     /**
      * Used by storage pools which want to keep VMs' information
      * @return true if additional VM info is needed (intended for storage pools).

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -118,7 +118,7 @@ public interface StorageManager extends StorageService {
             "storage.pool.disk.wait",
             "Storage",
             "60",
-            "Timeout (in secs) for the storage pool disk (of managed pool) to become available in the host. Currently only supported for PowerFlex.",
+            "Timeout (in secs) for the storage pool disk (of managed pool) to become available in the host. Currently supported for PowerFlex only.",
             true,
             ConfigKey.Scope.StoragePool,
             null);
@@ -127,7 +127,7 @@ public interface StorageManager extends StorageService {
             "storage.pool.client.timeout",
             "Storage",
             "60",
-            "Timeout (in secs) for the storage pool client connection timeout (for managed pools). Currently only supported for PowerFlex.",
+            "Timeout (in secs) for the API client connection timeout of storage pool (for managed pools). Currently supported for PowerFlex only.",
             false,
             ConfigKey.Scope.StoragePool,
             null);
@@ -136,11 +136,20 @@ public interface StorageManager extends StorageService {
             "storage.pool.client.max.connections",
             "Storage",
             "100",
-            "Maximum connections for the storage pool client (for managed pools). Currently only supported for PowerFlex.",
+            "Maximum connections for the API client of storage pool (for managed pools). Currently supported for PowerFlex only.",
             false,
             ConfigKey.Scope.StoragePool,
             null);
 
+    ConfigKey<Integer> STORAGE_POOL_CONNECTED_CLIENTS_LIMIT = new ConfigKey<>(Integer.class,
+            "storage.pool.connected.clients.limit",
+            "Storage",
+            "-1",
+            "Maximum connected storage pool clients supported for the storage (for managed pools), <= 0 for unlimited (default: -1). Currently supported for PowerFlex only.",
+            true,
+            ConfigKey.Scope.StoragePool,
+            null);
+
     ConfigKey<String> STORAGE_POOL_IO_POLICY = new ConfigKey<>(String.class,
             "kvm.storage.pool.io.policy",
             "Storage",
@@ -252,6 +261,10 @@ static Boolean getFullCloneConfiguration(Long storeId) {
 
     boolean canPoolProvideStorageStats(StoragePool pool);
 
+    boolean poolProvidesCustomStorageStats(StoragePool pool);
+
+    Map<String, String> getCustomStorageStats(StoragePool pool);
+
     /**
      * Checks if a host has running VMs that are using its local storage pool.
      * @return true if local storage is active on the host
@@ -286,6 +299,8 @@ static Boolean getFullCloneConfiguration(Long storeId) {
 
     boolean canHostAccessStoragePool(Host host, StoragePool pool);
 
+    boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool);
+
     Host getHost(long hostId);
 
     Host updateSecondaryStorage(long secStorageId, String newUrl);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareStorageClientCommandWrapper.java

@@ -0,0 +1,52 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.PrepareStorageClientAnswer;
+import com.cloud.agent.api.PrepareStorageClientCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.utils.Ternary;
+
+@ResourceWrapper(handles = PrepareStorageClientCommand.class)
+public class LibvirtPrepareStorageClientCommandWrapper extends CommandWrapper<PrepareStorageClientCommand, Answer, LibvirtComputingResource> {
+
+    private static final Logger s_logger = Logger.getLogger(LibvirtPrepareStorageClientCommandWrapper.class);
+
+    @Override
+    public Answer execute(PrepareStorageClientCommand cmd, LibvirtComputingResource libvirtComputingResource) {
+        final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
+        Ternary<Boolean, Map<String, String>, String> prepareStorageClientResult = storagePoolMgr.prepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid(), cmd.getDetails());
+        if (!prepareStorageClientResult.first()) {
+            String msg = prepareStorageClientResult.third();
+            s_logger.debug("Unable to prepare storage client, due to: " + msg);
+            return new PrepareStorageClientAnswer(cmd, false, msg);
+        }
+        Map<String, String> details = prepareStorageClientResult.second();
+        return new PrepareStorageClientAnswer(cmd, true, details);
+    }
+}