Issue 2580 Skip billing of the shared storages and the ones with cert…

…ain tag (#2584) (cherry picked from commit 74d8097)
epam · Apr 1, 2022 · 2d6e7d4 · 2d6e7d4
1 parent d5e972a
commit 2d6e7d4
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 14 deletions.
diff --git a/...c/main/java/com/epam/pipeline/billingreportagent/service/impl/CloudPipelineAPIClient.java b/...c/main/java/com/epam/pipeline/billingreportagent/service/impl/CloudPipelineAPIClient.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+ * Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,16 +18,17 @@
 
 import com.epam.pipeline.client.pipeline.CloudPipelineAPI;
 import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder;
+import com.epam.pipeline.client.pipeline.RetryingCloudPipelineApiExecutor;
 import com.epam.pipeline.entity.cluster.InstanceType;
 import com.epam.pipeline.entity.cluster.NodeDisk;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
 import com.epam.pipeline.entity.metadata.MetadataEntry;
 import com.epam.pipeline.entity.pipeline.PipelineRun;
 
 import com.epam.pipeline.entity.region.AbstractCloudRegion;
+import com.epam.pipeline.entity.security.acl.AclClass;
 import com.epam.pipeline.entity.user.PipelineUser;
 import com.epam.pipeline.exception.PipelineResponseException;
-import com.epam.pipeline.utils.QueryUtils;
 import com.epam.pipeline.vo.EntityVO;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -39,43 +40,49 @@
 public class CloudPipelineAPIClient {
 
     private final CloudPipelineAPI cloudPipelineAPI;
+    private final RetryingCloudPipelineApiExecutor retryingApiExecutor;
 
     public CloudPipelineAPIClient(@Value("${cloud.pipeline.host}") String cloudPipelineHostUrl,
                                   @Value("${cloud.pipeline.token}") String cloudPipelineToken) {
         this.cloudPipelineAPI =
                 new CloudPipelineApiBuilder(0, 0, cloudPipelineHostUrl, cloudPipelineToken)
                         .buildClient();
+        this.retryingApiExecutor = new RetryingCloudPipelineApiExecutor();
     }
 
     public List<PipelineUser> loadAllUsers() {
-        return QueryUtils.execute(cloudPipelineAPI.loadAllUsers());
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadAllUsers());
     }
 
     public List<PipelineRun> loadAllPipelineRunsActiveInPeriod(final String from, final String to) {
-        return QueryUtils.execute(cloudPipelineAPI.loadRunsActivityStats(from, to));
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadRunsActivityStats(from, to));
     }
 
     public List<AbstractDataStorage> loadAllDataStorages() {
-        return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages());
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadAllDataStorages());
     }
 
     public List<InstanceType> loadAllInstanceTypesForRegion(final Long regionId) {
         try {
-            return QueryUtils.execute(cloudPipelineAPI.loadAllInstanceTypesForRegion(regionId));
+            return retryingApiExecutor.execute(cloudPipelineAPI.loadAllInstanceTypesForRegion(regionId));
         } catch (PipelineResponseException e) {
             return Collections.emptyList();
         }
     }
 
     public List<MetadataEntry> loadMetadataEntry(List<EntityVO> entities) {
-        return QueryUtils.execute(cloudPipelineAPI.loadFolderMetadata(entities));
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadFolderMetadata(entities));
     }
 
     public List<NodeDisk> loadNodeDisks(final String nodeId) {
-        return QueryUtils.execute(cloudPipelineAPI.loadNodeDisks(nodeId));
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadNodeDisks(nodeId));
     }
 
     public List<AbstractCloudRegion> loadAllCloudRegions() {
-        return QueryUtils.execute(cloudPipelineAPI.loadAllRegions());
+        return retryingApiExecutor.execute(cloudPipelineAPI.loadAllRegions());
+    }
+
+    public List<EntityVO> searchEntriesByMetadata(final AclClass entityClass, final String key, final String value) {
+        return retryingApiExecutor.execute(cloudPipelineAPI.searchMetadata(key, value, entityClass));
     }
 }
diff --git a/...src/main/java/com/epam/pipeline/billingreportagent/service/impl/loader/StorageLoader.java b/...src/main/java/com/epam/pipeline/billingreportagent/service/impl/loader/StorageLoader.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+ * Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,29 +20,42 @@
 import com.epam.pipeline.billingreportagent.service.EntityLoader;
 import com.epam.pipeline.billingreportagent.service.impl.CloudPipelineAPIClient;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
+import com.epam.pipeline.entity.security.acl.AclClass;
 import com.epam.pipeline.entity.user.PipelineUser;
+import com.epam.pipeline.vo.EntityVO;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 @Component
 public class StorageLoader implements EntityLoader<AbstractDataStorage> {
 
     private final CloudPipelineAPIClient apiClient;
+    private final String storageExcludeKey;
+    private final String storageExcludeValue;
 
-    public StorageLoader(final CloudPipelineAPIClient apiClient) {
+    public StorageLoader(final CloudPipelineAPIClient apiClient,
+                         @Value("${sync.storage.billing.exclude.metadata.key:Billing status}")
+                         final String storageExcludeKey,
+                         @Value("${sync.storage.billing.exclude.metadata.value:Exclude}")
+                         final String storageExcludeValue) {
         this.apiClient = apiClient;
+        this.storageExcludeKey = storageExcludeKey;
+        this.storageExcludeValue = storageExcludeValue;
     }
 
     @Override
     public List<EntityContainer<AbstractDataStorage>> loadAllEntities() {
         final Map<String, EntityWithMetadata<PipelineUser>> usersWithMetadata = prepareUsers(apiClient);
-
+        final Set<Long> storageIdsToIgnore = loadStorageIdsToIgnoreByTag();
         return apiClient.loadAllDataStorages()
                 .stream()
+                .filter(storage -> isStorageBillable(storage, storageIdsToIgnore))
                 .map(storage -> EntityContainer.<AbstractDataStorage>builder()
                         .entity(storage)
                         .owner(usersWithMetadata.get(storage.getOwner()))
@@ -55,4 +68,15 @@ public List<EntityContainer<AbstractDataStorage>> loadAllEntitiesActiveInPeriod(
                                                                                     final LocalDateTime to) {
         return loadAllEntities();
     }
+
+    private Set<Long> loadStorageIdsToIgnoreByTag() {
+        return apiClient.searchEntriesByMetadata(AclClass.DATA_STORAGE, storageExcludeKey, storageExcludeValue)
+            .stream()
+            .map(EntityVO::getEntityId)
+            .collect(Collectors.toSet());
+    }
+
+    private boolean isStorageBillable(final AbstractDataStorage storage, final Set<Long> storageIdsToIgnore) {
+        return !storage.isShared() && !storageIdsToIgnore.contains(storage.getId());
+    }
 }
diff --git a/billing-report-agent/src/main/resources/application.properties b/billing-report-agent/src/main/resources/application.properties
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+# Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -51,3 +51,5 @@ sync.storage.file.index.pattern=*cp-%s-file-%d
 sync.storage.azure.auth.file=
 sync.storage.azure.offer.id=
 sync.aws.json.price.endpoint.template=https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json
+sync.storage.billing.exclude.metadata.key=Billing status
+sync.storage.billing.exclude.metadata.value=Exclude
diff --git a/deploy/docker/cp-billing-srv/config/application.properties b/deploy/docker/cp-billing-srv/config/application.properties
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+# Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,3 +47,7 @@ sync.storage.azure-blob.disable=${CP_BILLING_DISABLE_AZURE_BLOB:false}
 sync.storage.index.mapping=classpath:/templates/storage_billing.json
 sync.storage.index.name=storage-
 sync.storage.price.load.mode=${CP_BILLING_AWS_PRICE_TYPE:json}
+sync.storage.file.index.pattern=${CP_BILLING_STORAGE_INDEX_PATTERN:*cp-%s-%s-%d}
+sync.aws.json.price.endpoint.template=${CP_BILLING_AWS_PRICES_ENDPOINT:https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json}
+sync.storage.billing.exclude.metadata.key=${CP_BILLING_STORAGE_EXCLUDE_METADATA_KEY:Billing status}
+sync.storage.billing.exclude.metadata.value=${CP_BILLING_STORAGE_EXCLUDE_METADATA_VALUE:Exclude}