Issue 2580 Skip billing of the shared storages and the ones with cert…

…ain tag (#2584)
epam · Oct 19, 2023 · 4acb484 · 4acb484
1 parent 922f737
commit 4acb484
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 6 deletions.
diff --git a/...c/main/java/com/epam/pipeline/billingreportagent/service/impl/CloudPipelineAPIClient.java b/...c/main/java/com/epam/pipeline/billingreportagent/service/impl/CloudPipelineAPIClient.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+ * Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@
 import com.epam.pipeline.entity.pipeline.PipelineRun;
 
 import com.epam.pipeline.entity.region.AbstractCloudRegion;
+import com.epam.pipeline.entity.security.acl.AclClass;
 import com.epam.pipeline.entity.user.PipelineUser;
 import com.epam.pipeline.exception.PipelineResponseException;
 import com.epam.pipeline.utils.QueryUtils;
@@ -78,4 +79,8 @@ public List<NodeDisk> loadNodeDisks(final String nodeId) {
     public List<AbstractCloudRegion> loadAllCloudRegions() {
         return QueryUtils.execute(cloudPipelineAPI.loadAllRegions());
     }
+
+    public List<EntityVO> searchEntriesByMetadata(final AclClass entityClass, final String key, final String value) {
+        return QueryUtils.execute(cloudPipelineAPI.searchMetadata(key, value, entityClass));
+    }
 }
diff --git a/...src/main/java/com/epam/pipeline/billingreportagent/service/impl/loader/StorageLoader.java b/...src/main/java/com/epam/pipeline/billingreportagent/service/impl/loader/StorageLoader.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+ * Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,29 +20,42 @@
 import com.epam.pipeline.billingreportagent.service.EntityLoader;
 import com.epam.pipeline.billingreportagent.service.impl.CloudPipelineAPIClient;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
+import com.epam.pipeline.entity.security.acl.AclClass;
 import com.epam.pipeline.entity.user.PipelineUser;
+import com.epam.pipeline.vo.EntityVO;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 @Component
 public class StorageLoader implements EntityLoader<AbstractDataStorage> {
 
     private final CloudPipelineAPIClient apiClient;
+    private final String storageExcludeKey;
+    private final String storageExcludeValue;
 
-    public StorageLoader(final CloudPipelineAPIClient apiClient) {
+    public StorageLoader(final CloudPipelineAPIClient apiClient,
+                         @Value("${sync.storage.billing.exclude.metadata.key:Billing status}")
+                         final String storageExcludeKey,
+                         @Value("${sync.storage.billing.exclude.metadata.value:Exclude}")
+                         final String storageExcludeValue) {
         this.apiClient = apiClient;
+        this.storageExcludeKey = storageExcludeKey;
+        this.storageExcludeValue = storageExcludeValue;
     }
 
     @Override
     public List<EntityContainer<AbstractDataStorage>> loadAllEntities() {
         final Map<String, EntityWithMetadata<PipelineUser>> usersWithMetadata = prepareUsers(apiClient);
-
+        final Set<Long> storageIdsToIgnore = loadStorageIdsToIgnoreByTag();
         return apiClient.loadAllDataStorages()
                 .stream()
+                .filter(storage -> isStorageBillable(storage, storageIdsToIgnore))
                 .map(storage -> EntityContainer.<AbstractDataStorage>builder()
                         .entity(storage)
                         .owner(usersWithMetadata.get(storage.getOwner()))
@@ -55,4 +68,15 @@ public List<EntityContainer<AbstractDataStorage>> loadAllEntitiesActiveInPeriod(
                                                                                     final LocalDateTime to) {
         return loadAllEntities();
     }
+
+    private Set<Long> loadStorageIdsToIgnoreByTag() {
+        return apiClient.searchEntriesByMetadata(AclClass.DATA_STORAGE, storageExcludeKey, storageExcludeValue)
+            .stream()
+            .map(EntityVO::getEntityId)
+            .collect(Collectors.toSet());
+    }
+
+    private boolean isStorageBillable(final AbstractDataStorage storage, final Set<Long> storageIdsToIgnore) {
+        return !storage.isShared() && !storageIdsToIgnore.contains(storage.getId());
+    }
 }
diff --git a/billing-report-agent/src/main/resources/application.properties b/billing-report-agent/src/main/resources/application.properties
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+# Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -51,3 +51,5 @@ sync.storage.file.index.pattern=*cp-%s-file-%d
 sync.storage.azure.auth.file=
 sync.storage.azure.offer.id=
 sync.aws.json.price.endpoint.template=https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json
+sync.storage.billing.exclude.metadata.key=Billing status
+sync.storage.billing.exclude.metadata.value=Exclude
diff --git a/deploy/docker/cp-billing-srv/config/application.properties b/deploy/docker/cp-billing-srv/config/application.properties
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+# Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,3 +47,7 @@ sync.storage.azure-blob.disable=${CP_BILLING_DISABLE_AZURE_BLOB:false}
 sync.storage.index.mapping=classpath:/templates/storage_billing.json
 sync.storage.index.name=storage-
 sync.storage.price.load.mode=${CP_BILLING_AWS_PRICE_TYPE:json}
+
+sync.storage.billing.exclude.metadata.key=${CP_BILLING_STORAGE_EXCLUDE_METADATA_KEY:Billing status}
+sync.storage.billing.exclude.metadata.value=${CP_BILLING_STORAGE_EXCLUDE_METADATA_VALUE:Exclude}
+