Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extract the data-prepper-plugin-framework from data-prepper-core #4260

Merged
merged 3 commits into from
Mar 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion build-resources.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,6 @@ ext.coreProjects = [
project(':data-prepper-pipeline-parser'),
project(':data-prepper-plugins'),
project(':data-prepper-test-common'),
project(':data-prepper-test-event')
project(':data-prepper-test-event'),
project(':data-prepper-plugin-framework')
]
3 changes: 2 additions & 1 deletion data-prepper-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ dependencies {
implementation project(':data-prepper-plugins:common')
implementation project(':data-prepper-logstash-configuration')
implementation project(':data-prepper-pipeline-parser')
implementation project(':data-prepper-plugin-framework')
testImplementation project(':data-prepper-plugin-framework').sourceSets.test.output
testImplementation project(':data-prepper-plugins:common').sourceSets.test.output
implementation 'com.fasterxml.jackson.core:jackson-databind'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
Expand All @@ -28,7 +30,6 @@ dependencies {
implementation 'io.micrometer:micrometer-registry-cloudwatch2'
implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1'
implementation 'software.amazon.awssdk:cloudwatch'
implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
implementation platform('org.apache.logging.log4j:log4j-bom:2.22.1')
implementation 'org.apache.logging.log4j:log4j-core'
implementation 'org.apache.logging.log4j:log4j-slf4j2-impl'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@
import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel;
import org.opensearch.dataprepper.model.configuration.PluginSetting;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;
import org.opensearch.dataprepper.plugins.TestObjectPlugin;
import org.opensearch.dataprepper.plugins.TestPlugin;
import org.opensearch.dataprepper.plugins.test.TestPlugin;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;

import java.util.HashMap;
Expand All @@ -39,7 +38,7 @@ class DefaultPluginFactoryIT {
@Mock
private PipelinesDataFlowModel pipelinesDataFlowModel;
@Mock
private DataPrepperConfiguration dataPrepperConfiguration;
private ExtensionsConfiguration extensionsConfiguration;
private String pluginName;
private String objectPluginName;
private String pipelineName;
Expand All @@ -62,7 +61,7 @@ private DefaultPluginFactory createObjectUnderTest() {
coreContext.scan(DefaultAcknowledgementSetManager.class.getPackage().getName());
coreContext.scan(DefaultPluginFactory.class.getPackage().getName());
coreContext.register(PluginBeanFactoryProvider.class);
coreContext.registerBean(DataPrepperConfiguration.class, () -> dataPrepperConfiguration);
coreContext.registerBean(ExtensionsConfiguration.class, () -> extensionsConfiguration);
coreContext.registerBean(PipelinesDataFlowModel.class, () -> pipelinesDataFlowModel);
coreContext.refresh();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugin;
package org.opensearch.dataprepper.plugins;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.hamcrest.MatcherAssert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -25,8 +26,9 @@
import org.opensearch.dataprepper.parser.config.PipelineParserConfiguration;
import org.opensearch.dataprepper.peerforwarder.PeerForwarderProvider;
import org.opensearch.dataprepper.pipeline.router.RouterFactory;
import org.opensearch.dataprepper.plugins.TestPluginUsingExtension;
import org.opensearch.dataprepper.plugins.TestPluginUsingExtensionWithConfig;
import org.opensearch.dataprepper.plugin.DefaultPluginFactory;
import org.opensearch.dataprepper.plugin.ObjectMapperConfiguration;
import org.opensearch.dataprepper.plugin.TestPluggableInterface;
import org.opensearch.dataprepper.plugins.test.TestExtension;
import org.opensearch.dataprepper.sourcecoordination.SourceCoordinatorFactory;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
Expand Down Expand Up @@ -114,7 +116,7 @@ void tearDown() {

@Test
void applyExtensions_creates_a_single_instance_of_the_extension() {
assertThat(TestExtension.getConstructedInstances(), equalTo(1));
MatcherAssert.assertThat(TestExtension.getConstructedInstances(), equalTo(1));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.opensearch.dataprepper.model.configuration.PluginModel;
import org.opensearch.dataprepper.model.types.ByteCount;
import org.opensearch.dataprepper.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.opensearch.dataprepper.parser.config.MetricTagFilter;
import org.opensearch.dataprepper.peerforwarder.PeerForwarderConfiguration;
import org.opensearch.dataprepper.pipeline.PipelineShutdownOption;
import org.opensearch.dataprepper.plugin.ExtensionsConfiguration;

import java.time.Duration;
import java.util.Collections;
Expand All @@ -28,7 +29,7 @@
/**
* Class to hold configuration for DataPrepper, including server port and Log4j settings
*/
public class DataPrepperConfiguration {
public class DataPrepperConfiguration implements ExtensionsConfiguration {
static final Duration DEFAULT_SHUTDOWN_DURATION = Duration.ofSeconds(30L);

private static final String DEFAULT_SOURCE_COORDINATION_STORE = "in_memory";
Expand Down Expand Up @@ -225,6 +226,7 @@ public PipelineShutdownOption getPipelineShutdown() {
return pipelineShutdown;
}

@Override
public PipelineExtensions getPipelineExtensions() {
return pipelineExtensions;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,6 @@ public class TestDataProvider {
public static final String VALID_PEER_FORWARDER_WITH_ACM_SSL_CONFIG_FILE = "src/test/resources/valid_peer_forwarder_config_with_acm_ssl.yml";
public static final String VALID_DATA_PREPPER_CONFIG_WITH_METRIC_FILTER = "src/test/resources/valid_data_prepper_config_with_metric_filter.yml";
public static final String INVALID_DATA_PREPPER_CONFIG_WITH_METRIC_FILTER = "src/test/resources/invalid_data_prepper_config_with_metric_filter.yml";




public static Set<String> VALID_MULTIPLE_PIPELINE_NAMES = new HashSet<>(Arrays.asList("test-pipeline-1",
"test-pipeline-2", "test-pipeline-3"));
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

import org.opensearch.dataprepper.TestDataProvider;
import org.opensearch.dataprepper.model.types.ByteCount;
import org.opensearch.dataprepper.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.peerforwarder.PeerForwarderConfiguration;
import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.opensearch.dataprepper.model.types.ByteCount;
import org.opensearch.dataprepper.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;

import java.io.IOException;
import java.io.InputStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import org.opensearch.dataprepper.TestDataProvider;
import org.opensearch.dataprepper.peerforwarder.discovery.DiscoveryMode;
import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
import org.opensearch.dataprepper.pipeline.router.Router;
import org.opensearch.dataprepper.pipeline.router.RouterCopyRecordStrategy;
import org.opensearch.dataprepper.pipeline.router.RouterGetRecordStrategy;
import org.opensearch.dataprepper.plugins.TestSink;
import org.opensearch.dataprepper.plugins.TestSource;
import org.opensearch.dataprepper.plugins.test.TestSink;
import org.opensearch.dataprepper.plugins.test.TestSource;
import org.opensearch.dataprepper.plugins.TestSourceWithCoordination;
import org.opensearch.dataprepper.plugins.TestSourceWithEnhancedCoordination;
import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer;
Expand Down
5 changes: 5 additions & 0 deletions data-prepper-pipeline-parser/build.gradle
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

dependencies {
implementation project(':data-prepper-api')
implementation project(':data-prepper-plugins:blocking-buffer')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.parser;
package org.opensearch.dataprepper.pipeline.parser;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should look to see if we can make this package protected. Perhaps in another PR. It might involve moving a few other classes to reside in the same package.


import com.fasterxml.jackson.core.JacksonException;
import com.fasterxml.jackson.core.JsonParser;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.parser;
package org.opensearch.dataprepper.pipeline.parser;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should look to see if we can make this package protected. Perhaps in another PR. It might involve moving a few other classes to reside in the same package.


import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.parser;
package org.opensearch.dataprepper.pipeline.parser;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.parser;
package org.opensearch.dataprepper.pipeline.parser;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
Expand All @@ -14,8 +14,8 @@

import java.time.Duration;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;

public class DataPrepperDurationDeserializerTest {
Expand Down
22 changes: 22 additions & 0 deletions data-prepper-plugin-framework/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

dependencies {
implementation project(':data-prepper-api')
implementation project(':data-prepper-pipeline-parser')
implementation libs.commons.lang3
implementation 'javax.inject:javax.inject:1'
implementation 'javax.annotation:javax.annotation-api:1.3.2'
implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
implementation(libs.spring.core) {
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation(libs.spring.context) {
exclude group: 'commons-logging', module: 'commons-logging'
}
implementation libs.reflections.core
implementation 'com.fasterxml.jackson.core:jackson-databind'
testImplementation testLibs.mockito.inline
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
package org.opensearch.dataprepper.plugin;

import org.opensearch.dataprepper.model.configuration.PluginSetting;
import org.opensearch.dataprepper.model.plugin.PluginConfigObserver;
import org.opensearch.dataprepper.model.plugin.PluginConfigObservable;
import org.opensearch.dataprepper.model.plugin.PluginConfigObserver;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
package org.opensearch.dataprepper.plugin;

import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel;
import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration;

import javax.inject.Inject;
import javax.inject.Named;
Expand All @@ -21,10 +20,10 @@ public class ExtensionPluginConfigurationResolver {
private final Map<String, Object> dataPrepperConfigExtensionMap;

@Inject
public ExtensionPluginConfigurationResolver(final DataPrepperConfiguration dataPrepperConfiguration,
public ExtensionPluginConfigurationResolver(final ExtensionsConfiguration extensionsConfiguration,
final PipelinesDataFlowModel pipelinesDataFlowModel) {
this.dataPrepperConfigExtensionMap = dataPrepperConfiguration.getPipelineExtensions() == null?
new HashMap<>() : new HashMap<>(dataPrepperConfiguration.getPipelineExtensions().getExtensionMap());
this.dataPrepperConfigExtensionMap = extensionsConfiguration.getPipelineExtensions() == null?
new HashMap<>() : new HashMap<>(extensionsConfiguration.getPipelineExtensions().getExtensionMap());
combinedExtensionMap = new HashMap<>(dataPrepperConfigExtensionMap);
if (pipelinesDataFlowModel.getPipelineExtensions() != null) {
combinedExtensionMap.putAll(pipelinesDataFlowModel.getPipelineExtensions().getExtensionMap());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.opensearch.dataprepper.plugin;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably change this package to be something else. But, maybe in another PR.


import org.opensearch.dataprepper.model.configuration.PipelineExtensions;

public interface ExtensionsConfiguration {
PipelineExtensions getPipelineExtensions();
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.opensearch.dataprepper.model.types.ByteCount;
import org.opensearch.dataprepper.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer;
import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer;
import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer;
import org.springframework.context.annotation.Bean;

import javax.inject.Named;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
package org.opensearch.dataprepper.plugin;

import com.fasterxml.jackson.core.type.TypeReference;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.configuration.PluginSetting;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.validation.ConstraintViolation;
import jakarta.validation.Validator;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.configuration.PluginSetting;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import org.springframework.context.annotation.DependsOn;

import javax.inject.Named;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@

package org.opensearch.dataprepper.plugin;

import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.sink.Sink;
import org.opensearch.dataprepper.model.source.Source;
import org.opensearch.dataprepper.plugins.TestSink;
import org.opensearch.dataprepper.plugins.TestSource;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.sink.Sink;
import org.opensearch.dataprepper.model.source.Source;
import org.opensearch.dataprepper.plugins.test.TestSink;
import org.opensearch.dataprepper.plugins.test.TestSource;
import org.reflections.Reflections;

import java.util.Arrays;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,20 @@

package org.opensearch.dataprepper.plugin;

import org.hamcrest.MatcherAssert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.configuration.PipelineDescription;
import org.opensearch.dataprepper.model.configuration.PluginSetting;
import org.opensearch.dataprepper.model.plugin.NoPluginFoundException;
import org.opensearch.dataprepper.model.plugin.PluginConfigObservable;
import org.opensearch.dataprepper.model.sink.Sink;
import org.opensearch.dataprepper.plugins.TestSink;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.opensearch.dataprepper.plugins.test.TestSink;
import org.springframework.beans.factory.BeanFactory;

import java.util.ArrayList;
Expand Down Expand Up @@ -396,7 +397,7 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found_with_corr
.willReturn(expectedInstance);

assertThat(createObjectUnderTest().loadPlugin(baseClass, pluginSetting), equalTo(expectedInstance));
assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).deprecatedName(), equalTo(TEST_SINK_DEPRECATED_NAME));
MatcherAssert.assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).deprecatedName(), equalTo(TEST_SINK_DEPRECATED_NAME));
verify(beanFactoryProvider).get();
}
}
Expand Down
Loading
Loading