Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding Apache NiFi nifi-commons (nifi/nifi-commons) to openctest #35

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Adding Apache NiFi nifi-commons (nifi/nifi-commons) to openctest #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ef7812a
modify for nifi
Dec 6, 2022
705c63a
update nifi setup
Dec 10, 2022
9c22266
add params
Dec 11, 2022
09c6e0d
modify identify runner
Dec 11, 2022
09f1f7f
update add project
Dec 11, 2022
9406529
finish nifi
Dec 12, 2022
fee78df
push patch
Dec 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion core/add_project.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ function setup_alluxio() {
cd core
mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
}
function setup_nifi(){
[ ! -d "app/ctest-nifi" ] && git clone https://github.com/lilacyl/nifi.git app/ctest-nifi
cd app/ctest-nifi
git fetch && git checkout ctest-injection
mvn clean install -pl nifi-commons/ -DskipTest
}

function usage() {
echo "Usage: add_project.sh <main project>"
Expand All @@ -64,7 +70,8 @@ function main() {
hbase) setup_hbase ;;
zookeeper) setup_zookeeper ;;
alluxio) setup_alluxio ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
nifi) setup_nifi ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, nifi, and alluxio." ;;
esac
fi
}
Expand Down
12 changes: 11 additions & 1 deletion core/ctest_const.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,21 @@
HBASE = "hbase-server"
ZOOKEEPER = "zookeeper-server"
ALLUXIO = "alluxio-core"
NIFI = "nifi-commons"

CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
CTEST_NIFI_DIR = os.path.join(APP_DIR, "ctest-nifi")

PROJECT_DIR = {
HCOMMON: CTEST_HADOOP_DIR,
HDFS: CTEST_HADOOP_DIR,
HBASE: CTEST_HBASE_DIR,
ZOOKEEPER: CTEST_ZK_DIR,
ALLUXIO: CTEST_ALLUXIO_DIR,
NIFI: CTEST_NIFI_DIR,
}


Expand All @@ -34,12 +37,14 @@
HBASE: "hbase-server",
ZOOKEEPER: "zookeeper-server",
ALLUXIO: "core",
NIFI: "nifi-commons",
}


# surefire report
SUREFIRE_SUBDIR = "target/surefire-reports/"
SUREFIRE_XML = "TEST-{}.xml" # slot is the classname
SUREFIRE_XML_NIFI = "TEST-org.apache.nifi.util.{}.xml" # slot is the classname
SUREFIRE_TXT = "{}.txt" # testclass
SUREFIRE_OUTTXT = "{}-output.txt" #testclass

Expand All @@ -58,6 +63,7 @@
os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
],
NIFI: [os.path.join(CTEST_NIFI_DIR, MODULE_SUBDIR[NIFI],"nifi-properties", SUREFIRE_SUBDIR)],
}

# default or deprecate conf path
Expand All @@ -74,7 +80,8 @@
HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
NIFI: os.path.join(DEFAULT_CONF_DIR, NIFI + "-default.tsv"),
}


Expand All @@ -96,6 +103,9 @@
],
ALLUXIO: [
os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
],
NIFI: [
os.path.join(CTEST_NIFI_DIR, "nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties")
]
}

Expand Down
229 changes: 229 additions & 0 deletions core/default_configs/nifi-commons-default.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
nifi.properties.file.path
nifi.flow.configuration.file
nifi.flow.configuration.json.file
nifi.flow.configuration.archive.enabled "true"
nifi.flow.configuration.archive.dir
nifi.flow.configuration.archive.max.time "30 days"
nifi.flow.configuration.archive.max.storage "500 MB"
nifi.flow.configuration.archive.max.count
nifi.authorizer.configuration.file "conf/authorizers.xml"
nifi.login.identity.provider.configuration.file "conf/login-identity-providers.xml"
nifi.database.directory
nifi.restore.directory
nifi.flowservice.writedelay.interval
nifi.flowcontroller.autoResumeState true
nifi.flowcontroller.graceful.shutdown.period
nifi.nar.library.directory
nifi.nar.library.directory.
nifi.nar.library.autoload.directory
nifi.nar.working.directory
nifi.nar.unpack.uber.jar false
nifi.documentation.working.directory "./work/docs/components"
nifi.sensitive.props.key
nifi.sensitive.props.algorithm
nifi.h2.url.append
nifi.remote.input.host
nifi.remote.input.socket.port null
nifi.remote.input.secure
nifi.remote.input.http.enabled
nifi.remote.input.http.transaction.ttl "30 secs"
nifi.remote.contents.cache.expiration
nifi.templates.directory Paths.get("conf", "templates")
nifi.administrative.yield.duration "30 sec"
nifi.bored.yield.duration "10 millis"
nifi.processor.scheduling.timeout
nifi.queue.backpressure.count 10_000L
nifi.queue.backpressure.size "1 GB"
nifi.listener.bootstrap.port 0
nifi.repository.encryption.protocol.version
nifi.repository.encryption.key.id
nifi.repository.encryption.key.provider
nifi.repository.encryption.key.provider.keystore.location
nifi.repository.encryption.key.provider.keystore.password
nifi.content.repository.directory.
nifi.content.repository.implementation
nifi.content.claim.max.appendable.size "1 MB"
nifi.content.claim.max.flow.files 100
nifi.content.repository.archive.max.retention.period
nifi.content.repository.archive.max.usage.percentage
nifi.content.repository.archive.backpressure.percentage
nifi.content.repository.archive.enabled
nifi.content.repository.archive.cleanup.frequency
nifi.content.viewer.url
nifi.content.repository.encryption.key
nifi.content.repository.encryption.key.id
nifi.content.repository.encryption.key.provider.implementation
nifi.content.repository.encryption.key.provider.location
nifi.content.repository.encryption.key.provider.password
nifi.flowfile.repository.implementation
nifi.flowfile.repository.wal.implementation
nifi.flowfile.repository.always.sync
nifi.flowfile.repository.directory
nifi.flowfile.repository.checkpoint.interval
nifi.flowfile.repository.encryption.key
nifi.flowfile.repository.encryption.key.id
nifi.flowfile.repository.encryption.key.provider.implementation
nifi.flowfile.repository.encryption.key.provider.location
nifi.flowfile.repository.encryption.key.provider.password
nifi.swap.manager.implementation
nifi.queue.swap.threshold 20000
nifi.provenance.repository.implementation
nifi.provenance.repository.directory.
nifi.provenance.repository.max.storage.time
nifi.provenance.repository.max.storage.size
nifi.provenance.repository.rollover.time
nifi.provenance.repository.rollover.size
nifi.provenance.repository.rollover.events
nifi.provenance.repository.query.threads
nifi.provenance.repository.index.threads
nifi.provenance.repository.compress.on.rollover
nifi.provenance.repository.indexed.fields
nifi.provenance.repository.indexed.attributes
nifi.provenance.repository.index.shard.size
nifi.provenance.repository.journal.count
nifi.provenance.repository.encryption.key
nifi.provenance.repository.encryption.key.id
nifi.provenance.repository.encryption.key.provider.implementation
nifi.provenance.repository.encryption.key.provider.location
nifi.provenance.repository.encryption.key.provider.password
nifi.provenance.repository.debug.frequency
nifi.components.status.repository.implementation
nifi.components.status.snapshot.frequency "5 mins"
nifi.status.repository.questdb.persist.node.days
nifi.status.repository.questdb.persist.component.days
nifi.status.repository.questdb.persist.location
nifi.security.keystore
nifi.security.keystoreType
nifi.security.keystorePasswd
nifi.security.keyPasswd
nifi.security.truststore
nifi.security.truststoreType
nifi.security.truststorePasswd
nifi.security.autoreload.enabled
nifi.security.autoreload.interval "10 secs"
nifi.security.user.authorizer
nifi.security.allow.anonymous.authentication
nifi.security.user.login.identity.provider
nifi.security.ocsp.responder.url
nifi.security.ocsp.responder.certificate
nifi.security.identity.mapping.pattern.
nifi.security.identity.mapping.value.
nifi.security.identity.mapping.transform.
nifi.security.group.mapping.pattern.
nifi.security.group.mapping.value.
nifi.security.group.mapping.transform.
nifi.security.user.jws.key.rotation.period
nifi.security.user.oidc.discovery.url
nifi.security.user.oidc.connect.timeout "5 secs"
nifi.security.user.oidc.read.timeout "5 secs"
nifi.security.user.oidc.client.id
nifi.security.user.oidc.client.secret
nifi.security.user.oidc.truststore.strategy "JDK"
nifi.security.user.oidc.preferred.jwsalgorithm
nifi.security.user.oidc.additional.scopes
nifi.security.user.oidc.claim.identifying.user
nifi.security.user.oidc.claim.groups
nifi.security.user.oidc.fallback.claims.identifying.user
nifi.security.user.knox.url
nifi.security.user.knox.publicKey
nifi.security.user.knox.cookieName
nifi.security.user.knox.audiences
nifi.security.user.saml.idp.metadata.url
nifi.security.user.saml.sp.entity.id
nifi.security.user.saml.identity.attribute.name
nifi.security.user.saml.group.attribute.name
nifi.security.user.saml.metadata.signing.enabled "false"
nifi.security.user.saml.request.signing.enabled "false"
nifi.security.user.saml.want.assertions.signed "true"
nifi.security.user.saml.signature.algorithm "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"
nifi.security.user.saml.signature.digest.algorithm
nifi.security.user.saml.message.logging.enabled "false"
nifi.security.user.saml.authentication.expiration "12 hours"
nifi.security.user.saml.single.logout.enabled "false"
nifi.security.user.saml.http.client.truststore.strategy "JDK"
nifi.security.user.saml.http.client.connect.timeout "30 secs"
nifi.security.user.saml.http.client.read.timeout "30 secs"
nifi.web.http.port
nifi.web.http.port.forwarding
nifi.web.http.host
nifi.web.http.network.interface.
nifi.web.https.port
nifi.web.https.port.forwarding
nifi.web.https.host
nifi.web.https.application.protocols
nifi.web.https.ciphersuites.include
nifi.web.https.ciphersuites.exclude
nifi.web.https.network.interface.
nifi.web.jetty.working.directory "./work/jetty"
nifi.web.jetty.threads 200
nifi.web.max.header.size "16 KB"
nifi.web.proxy.context.path
nifi.web.proxy.host
nifi.web.max.content.size "20 MB"
nifi.web.max.requests.per.second 30000
nifi.web.max.access.token.requests.per.second 25
nifi.web.request.timeout "60 secs"
nifi.web.request.ip.whitelist
nifi.web.should.send.server.version "true"
nifi.web.request.log.format
nifi.ui.banner.text
nifi.ui.autorefresh.interval
nifi.cluster.protocol.heartbeat.interval "5 sec"
nifi.cluster.protocol.heartbeat.missable.max 8
nifi.cluster.protocol.is.secure
nifi.cluster.is.node
nifi.cluster.node.address
nifi.cluster.node.protocol.port
nifi.cluster.node.protocol.max.threads 50
nifi.cluster.node.connection.timeout "5 sec"
nifi.cluster.node.read.timeout "5 sec"
nifi.cluster.node.max.concurrent.requests 100
nifi.cluster.firewall.file
nifi.cluster.flow.election.max.wait.time "5 mins"
nifi.cluster.flow.election.max.candidates
nifi.cluster.load.balance.host
nifi.cluster.load.balance.port 6342
nifi.cluster.load.balance.connections.per.node 4
nifi.cluster.load.balance.max.thread.count 8
nifi.cluster.load.balance.comms.timeout "30 sec"
nifi.zookeeper.connect.string
nifi.zookeeper.connect.timeout "3 secs"
nifi.zookeeper.session.timeout "3 secs"
nifi.zookeeper.root.node "/nifi"
nifi.zookeeper.client.secure false
nifi.zookeeper.security.keystore
nifi.zookeeper.security.keystoreType
nifi.zookeeper.security.keystorePasswd
nifi.zookeeper.security.truststore
nifi.zookeeper.security.truststoreType
nifi.zookeeper.security.truststorePasswd
nifi.zookeeper.auth.type "default"
nifi.zookeeper.kerberos.removeHostFromPrincipal "true"
nifi.zookeeper.kerberos.removeRealmFromPrincipal "true"
nifi.zookeeper.jute.maxbuffer 1048575
nifi.kerberos.krb5.file
nifi.kerberos.service.principal
nifi.kerberos.service.keytab.location
nifi.kerberos.spnego.principal
nifi.kerberos.spnego.keytab.location
nifi.kerberos.spnego.authentication.expiration "12 hours"
nifi.state.management.configuration.file "conf/state-management.xml"
nifi.state.management.provider.local
nifi.state.management.provider.cluster
nifi.state.management.embedded.zookeeper.start
nifi.state.management.embedded.zookeeper.properties
nifi.variable.registry.properties
nifi.analytics.predict.enabled "false"
nifi.analytics.predict.interval "3 mins"
nifi.analytics.query.interval "3 mins"
nifi.analytics.connection.model.implementation
nifi.analytics.connection.model.score.name
nifi.analytics.connection.model.score.threshold
nifi.monitor.long.running.task.schedule
nifi.monitor.long.running.task.threshold
nifi.diagnostics.on.shutdown.enabled
nifi.diagnostics.on.shutdown.verbose
nifi.diagnostics.on.shutdown.directory "./diagnostics"
nifi.diagnostics.on.shutdown.max.filecount 10
nifi.diagnostics.on.shutdown.max.directory.size "10 MB"
nifi.performance.tracking.percentage 0
1 change: 1 addition & 0 deletions core/generate_ctest/ctest_mapping/ctests-nifi-commons.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"nifi.ui.banner.text": ["NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidPortValue", "NiFiPropertiesTest#testProperties", "NiFiPropertiesTest#testBlankProperties", "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", "NiFiPropertiesTest#testValidateProperties", "NiFiPropertiesTest#testMissingProperties"], "nifi.nar.library.directory": ["NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidPortValue", "NiFiPropertiesTest#testProperties", "NiFiPropertiesTest#testBlankProperties", "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", "NiFiPropertiesTest#testValidateProperties", "NiFiPropertiesTest#testMissingProperties"]}
4 changes: 2 additions & 2 deletions core/generate_ctest/inject.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def inject_config(param_value_pairs):
for p, v in param_value_pairs.items():
print(">>>>[ctest_core] injecting {} with value {}".format(p, v))

if project in [ZOOKEEPER, ALLUXIO]:
if project in [ZOOKEEPER, ALLUXIO, NIFI]:
for inject_path in INJECTION_PATH[project]:
print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
file = open(inject_path, "w")
Expand All @@ -41,7 +41,7 @@ def inject_config(param_value_pairs):

def clean_conf_file(project):
print(">>>> cleaning injected configuration from file")
if project in [ZOOKEEPER, ALLUXIO]:
if project in [ZOOKEEPER, ALLUXIO, NIFI]:
for inject_path in INJECTION_PATH[project]:
file = open(inject_path, "w")
file.write("\n")
Expand Down
3 changes: 2 additions & 1 deletion core/generate_ctest/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ def test_value_pair(test_input):
mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")

associated_tests = mapping[param] if param in mapping else []
if len(mapping[param]) != 0:
print(param)
if param in mapping and len(mapping[param]) != 0:
for value in values:
tr = run_test_seperate(param, value, associated_tests)

Expand Down
5 changes: 4 additions & 1 deletion core/generate_ctest/parse_output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@ def parse_surefire(clsname, expected_methods):
try:
fpath = None
for surefire_path in SUREFIRE_DIR[project]:
xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname))
if project == "nifi-commons":
xml_path = os.path.join(surefire_path, SUREFIRE_XML_NIFI.format(clsname))
else:
xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname))
if os.path.exists(xml_path):
print(">>>>[ctest_core] surefire report path: " + xml_path)
fpath = open(xml_path)
Expand Down
6 changes: 3 additions & 3 deletions core/generate_ctest/program_input.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
# run mode
"run_mode": "generate_ctest", # string
# name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
"project": "hadoop-common", # string
"project": "nifi-commons", # string
# path to param -> tests json mapping
"mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
"mapping_path": "../../data/ctest_mapping/opensource-nifi-commons.json", # string
# good values of params tests will be run against
"param_value_tsv": "sample-hadoop-common.tsv", # string
"param_value_tsv": "sample-nifi-commons.tsv", # string
# display the terminal output live, without saving any results
"display_mode": False, # bool
# whether to use mvn test or mvn surefire:test
Expand Down
Loading