Skip to content

Commit

Permalink
#495 Add functionality for extending foundation-mda
Browse files Browse the repository at this point in the history
  • Loading branch information
carter-cundiff committed Dec 13, 2024
1 parent 6dc75d8 commit 97bc5e5
Show file tree
Hide file tree
Showing 62 changed files with 687 additions and 220 deletions.
6 changes: 5 additions & 1 deletion DRAFT_RELEASE_NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ Data access through [GraphQL](https://graphql.org/) has been deprecated and repl
# Breaking Changes
_Note: instructions for adapting to these changes are outlined in the upgrade instructions below._

- _There are no breaking changes in teh 1.11 release._
- The following Java classes have been renamed:
| Old Java Class | New Java Class |
|-------------------------------|------------------------------------|
| `AIOpsModelInstanceRepostory` | `AissembleModelInstanceRepository` |
| `AiopsMdaJsonUtils` | `AissembleMdaJsonUtils` |

# Known Issues
_There are no known issues with the 1.11 release._
Expand Down
4 changes: 2 additions & 2 deletions build-parent/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<version.buildhelper.plugin>3.6.0</version.buildhelper.plugin>
<version.buildnumber.plugin>3.2.1</version.buildnumber.plugin>
<version.failsafe.plugin>${version.maven.surefire.plugin}</version.failsafe.plugin>
<version.fermenter>2.10.3</version.fermenter>
<version.fermenter>2.10.5</version.fermenter>
<version.fermenter.legacy.tools>2.8.0</version.fermenter.legacy.tools>
<version.habushu.plugin>2.17.0</version.habushu.plugin>
<version.python>3.11.4</version.python>
Expand Down Expand Up @@ -386,7 +386,7 @@
<artifactId>fermenter-mda</artifactId>
<version>${version.fermenter}</version>
<configuration>
<metadataRepositoryImpl>com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory
<metadataRepositoryImpl>com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository
</metadataRepositoryImpl>
</configuration>
<executions>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import com.boozallen.aiops.mda.generator.common.PipelineImplementationEnum;
import com.boozallen.aiops.mda.generator.util.SemanticDataUtil;
import com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory;
import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.mda.maven.ArtifactType;
import com.boozallen.mda.maven.PipelineType;
import com.boozallen.mda.maven.util.ArtifactCopier;
Expand Down Expand Up @@ -45,6 +45,8 @@

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.nio.file.*;
import java.util.*;
Expand Down Expand Up @@ -118,6 +120,12 @@ public class PipelineArtifactsMojo extends AbstractMojo {
@Parameter(required = true)
private String habushuArtifactVersion;

/**
* Config option for setting the metadata repository when using an extension of AissembleModelInstanceRepository
*/
@Parameter(defaultValue = "com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository")
private String metadataRepositoryImpl;

@Parameter()
private String mavenDependencyPluginVersion;

Expand Down Expand Up @@ -166,6 +174,32 @@ public ModelRepositoryConfiguration setupConfig() {
return config;
}

/**
* Helper method to retrieve and validate the ModelInstanceRepository implementation class.
*
* @param config
* @return {@link AissembleModelInstanceRepository} repository
* @throws ClassNotFoundException
* @throws InvocationTargetException
* @throws NoSuchMethodException
* @throws InstantiationException
* @throws IllegalAccessException
* @throws MojoExecutionException
*/
protected AissembleModelInstanceRepository buildModelInstanceRepository(ModelRepositoryConfiguration config)
throws ClassNotFoundException,InvocationTargetException, NoSuchMethodException, InstantiationException,
IllegalAccessException, MojoExecutionException {
logger.info("Loading metamodel repository implementation: {}", metadataRepositoryImpl);

Class<?> repoImplClass = Class.forName(metadataRepositoryImpl);
Class<?>[] constructorParamTypes = { ModelRepositoryConfiguration.class };
Constructor<?> constructor = repoImplClass.getConstructor(constructorParamTypes);
Object[] params = { config };
AissembleModelInstanceRepository repository = (AissembleModelInstanceRepository) constructor.newInstance(params);
ModelInstanceRepositoryManager.setRepository(repository);

return repository;
}

/**
* Reads in the MDA models and calls the appropriate helper function based on the implementation type
Expand All @@ -180,9 +214,14 @@ public void execute() throws MojoExecutionException {
//caller helper function for setting up the config
ModelRepositoryConfiguration config = setupConfig();

//Create a model repository from the config
AIOpsModelInstanceRepostory repository = new AIOpsModelInstanceRepostory(config);
ModelInstanceRepositoryManager.setRepository(repository);
AissembleModelInstanceRepository repository;
try {
//Create a model repository from the config
repository = buildModelInstanceRepository(config);
} catch (ClassNotFoundException | NoSuchMethodException | InstantiationException |
IllegalAccessException | InvocationTargetException e) {
throw new MojoExecutionException("Could not successfully load metamodel repository", e);
}

//Load and validate the models
repository.load();
Expand Down Expand Up @@ -652,4 +691,8 @@ public void setDataRecordsDirectory(String dataRecordsDirectory) {
public void setDataRecordModules(List<String> dataRecordModules) {
this.dataRecordModules = dataRecordModules;
}

public void setMetadataRepositoryImpl(String metadataRepositoryImpl) {
this.metadataRepositoryImpl = metadataRepositoryImpl;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package com.boozallen.mda.maven;

import static org.junit.Assert.assertNull;

/*-
* #%L
* aiSSEMBLE::Foundation::Maven Plugins::MDA Maven Plugin
* %%
* Copyright (C) 2021 Booz Allen
* %%
* This software package is licensed under the Booz Allen Public License. All Rights Reserved.
* #L%
*/

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.commons.io.FileUtils;

import com.boozallen.mda.maven.mojo.PipelineArtifactsMojo;

import io.cucumber.java.After;
import io.cucumber.java.Before;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;

public class MetamodelExtensionTest {
private MojoTestCaseWrapper mojoTestCase = new MojoTestCaseWrapper();
private Exception exception;
private File modelSource;
private String modelRepositoryImpl;
private String testProject;

@Before("@MetamodelExtension")
public void setup() throws Exception {
mojoTestCase.configurePluginTestHarness();
Path testPom = Paths.get("src", "test", "resources", "test-pom", "pom.xml").toAbsolutePath();
Path testProject = Paths.get("target", "test-project").toAbsolutePath();
Files.createDirectories(testProject);
Files.copy(testPom, testProject.resolve("pom.xml"));
this.testProject = testProject.toString();
}

@After("@MetamodelExtension")
public void teardown() throws Exception {
mojoTestCase.tearDownPluginTestHarness();
FileUtils.deleteDirectory(Paths.get(this.testProject).toFile());
}

@Given("a model instance repository extending foundation-mda")
public void a_model_instance_repository_extending_foundation_mda() {
this.modelRepositoryImpl = "com.boozallen.mda.maven.metadata.repository.ExtensionModelInstanceRepository";
}

@Given("a pipeline with metamodel extensions")
public void a_pipeline_with_metamodel_extensions() {
this.modelSource = new File("src/test/resources/models/model-extension.jar");
}

@When("the copy-pipeline-artifacts goal is executed")
public void the_copy_pipeline_artifacts_goal_is_executed() {
//Read in the test pom for the correct pipeline type and configure the mojo with the parameters
File testPom = new File(testProject + "/pom.xml");

try {
PipelineArtifactsMojo mojo = (PipelineArtifactsMojo) mojoTestCase.lookupConfiguredMojo(testPom, "copy-pipeline-artifacts");
mojo.setModelsSource(this.modelSource);
mojo.setPipelinesDirectory(new File("src/test/resources/pipelines/data-flow").getAbsolutePath() + "/");
mojo.setMetadataRepositoryImpl(this.modelRepositoryImpl);
mojo.execute();
} catch (Exception exception) {
this.exception = exception;
}
}

@Then("the extended metamodel is read successfully")
public void the_extended_metamodel_is_read_successfully() {
assertNull("An exception occurred when executing the mojo with a metamodel repository extension", this.exception);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package com.boozallen.mda.maven.metadata.repository;

/*-
* #%L
* aiSSEMBLE::Foundation::Maven Plugins::MDA Maven Plugin
* %%
* Copyright (C) 2021 Booz Allen
* %%
* This software package is licensed under the Booz Allen Public License. All Rights Reserved.
* #L%
*/

import org.technologybrewery.fermenter.mda.metamodel.ModelRepositoryConfiguration;
import org.technologybrewery.fermenter.mda.util.JsonUtils;

import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.aiops.mda.metamodel.element.PipelineType;
import com.boozallen.aiops.mda.metamodel.element.PipelineTypeElement;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;

/*
* Class extending the aissemble metamodel repository with our custom classes
*/
public class ExtensionModelInstanceRepository extends AissembleModelInstanceRepository {
public static class PipelineTypeElementExtension extends PipelineTypeElement {
public String simpleField;
}

public ExtensionModelInstanceRepository(ModelRepositoryConfiguration config) {
super(config);
this.configureCustomObjectMapper();
}

public void configureCustomObjectMapper() {
SimpleModule module = new SimpleModule();

// Add custom Pipeline extension
module.addAbstractTypeMapping(PipelineType.class, PipelineTypeElementExtension.class);

ObjectMapper localMapper = new ObjectMapper();
localMapper.registerModule(module);
JsonUtils.setObjectMapper(localMapper);
}
}
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
@MetamodelExtension
Feature: As an aiSSEMBLE user, I want pipeline artifacts derived from MDA models with extended metamodel schemas

Scenario: Use model instance repository with metamodel extensions
Given a model instance repository extending foundation-mda
And a pipeline with metamodel extensions
When the copy-pipeline-artifacts goal is executed
Then the extended metamodel is read successfully
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@
*/

import com.boozallen.aiops.mda.generator.common.VelocityProperty;
import com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory;
import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.aiops.mda.metamodel.element.Pipeline;
import org.apache.velocity.VelocityContext;
import org.technologybrewery.fermenter.mda.generator.GenerationContext;
import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceRepositoryManager;

import java.util.Map;

Expand Down Expand Up @@ -44,19 +43,6 @@ protected void generateCommon(GenerationContext generationContext, String module
generateFile(generationContext, velocityContext);
}

/**
* Returns true if at least one pipeline is configured with a step that has
* data profiling enabled.
* @return
*/
protected boolean isDataProfilingEnabled() {
AIOpsModelInstanceRepostory metamodelRepository = ModelInstanceRepositoryManager
.getMetamodelRepository(AIOpsModelInstanceRepostory.class);

Map<String, Pipeline> pipelineMap = metamodelRepository.getPipelinesByContext(metadataContext);
return Pipeline.aPipelineExistsWhere(pipelineMap.values(), Pipeline::isDataProfilingEnabled);
}

/**
* Returns the artifact id for the module to generate.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import com.boozallen.aiops.mda.generator.common.PipelineEnum;
import com.boozallen.aiops.mda.generator.common.VelocityProperty;
import com.boozallen.aiops.mda.generator.util.PipelineUtils;
import com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory;
import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.aiops.mda.metamodel.element.BasePipelineDecorator;
import com.boozallen.aiops.mda.metamodel.element.BaseStepDecorator;
import com.boozallen.aiops.mda.metamodel.element.Pipeline;
Expand All @@ -23,7 +23,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.technologybrewery.fermenter.mda.generator.GenerationContext;
import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceRepositoryManager;
import org.apache.maven.model.Model;

import java.nio.file.Paths;
Expand All @@ -47,8 +46,7 @@ public abstract class AbstractLineageResourcesGenerator extends AbstractResource
@Override
public void generate(GenerationContext generationContext) {
VelocityContext vc = getNewVelocityContext(generationContext);
AIOpsModelInstanceRepostory metamodelRepository = ModelInstanceRepositoryManager
.getMetamodelRepository(AIOpsModelInstanceRepostory.class);
AissembleModelInstanceRepository metamodelRepository = (AissembleModelInstanceRepository) generationContext.getModelInstanceRepository();

Map<String, Pipeline> pipelineMap = metamodelRepository.getPipelinesByContext(metadataContext);
List<Pipeline> pipelines = new ArrayList<>(pipelineMap.values());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@
import com.boozallen.aiops.mda.generator.common.DataFlowStrategy;
import com.boozallen.aiops.mda.generator.common.MachineLearningStrategy;
import com.boozallen.aiops.mda.generator.common.VelocityProperty;
import com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory;
import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.aiops.mda.metamodel.element.Pipeline;
import org.apache.velocity.VelocityContext;
import org.technologybrewery.fermenter.mda.generator.GenerationContext;
import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceRepositoryManager;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -43,8 +42,7 @@ public class AirflowDockerPomGenerator extends AbstractMavenModuleGenerator {
*/
@Override
public void generate(GenerationContext context) {
AIOpsModelInstanceRepostory metamodelRepository = ModelInstanceRepositoryManager
.getMetamodelRepository(AIOpsModelInstanceRepostory.class);
AissembleModelInstanceRepository metamodelRepository = (AissembleModelInstanceRepository) context.getModelInstanceRepository();

Map<String, Pipeline> pipelineMap = metamodelRepository.getPipelinesByContext(metadataContext);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,11 @@

import com.boozallen.aiops.mda.generator.common.DataFlowStrategy;
import com.boozallen.aiops.mda.generator.common.VelocityProperty;
import com.boozallen.aiops.mda.metamodel.AIOpsModelInstanceRepostory;
import com.boozallen.aiops.mda.metamodel.AissembleModelInstanceRepository;
import com.boozallen.aiops.mda.metamodel.element.Pipeline;
import com.boozallen.aiops.mda.metamodel.element.BasePipelineDecorator;
import org.apache.velocity.VelocityContext;
import org.technologybrewery.fermenter.mda.generator.GenerationContext;
import org.technologybrewery.fermenter.mda.metamodel.ModelInstanceRepositoryManager;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -40,8 +39,7 @@ public class DataDeliveryDagGenerator extends AbstractPythonGenerator {
*/
@Override
public void generate(GenerationContext generationContext) {
AIOpsModelInstanceRepostory metamodelRepository = ModelInstanceRepositoryManager
.getMetamodelRepository(AIOpsModelInstanceRepostory.class);
AissembleModelInstanceRepository metamodelRepository = (AissembleModelInstanceRepository) generationContext.getModelInstanceRepository();

Map<String, Pipeline> pipelineMap = metamodelRepository.getPipelinesByContext(metadataContext);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ public class DataRecordsJavaPomGenerator extends DataRecordsPomGenerator {


@Override
protected boolean shouldGenerate() {
return SemanticDataUtil.areJavaDataRecordsNeeded(metadataContext);
protected boolean shouldGenerate(GenerationContext generationContext) {
return SemanticDataUtil.areJavaDataRecordsNeeded(generationContext, metadataContext);
}

@Override
Expand Down
Loading

0 comments on commit 97bc5e5

Please sign in to comment.