From d90f01287e7d15a4738d917dce9604e92017112a Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Wed, 20 Nov 2024 20:39:04 +0100 Subject: [PATCH 1/4] NIFI-14029 - BitBucket Registry Client --- .../nifi-atlassian-extensions/pom.xml | 52 ++ .../BitBucketAuthenticationType.java | 49 ++ .../BitBucketFlowRegistryClient.java | 163 +++++ .../bitbucket/BitBucketRepositoryClient.java | 576 ++++++++++++++++++ ...ache.nifi.registry.flow.FlowRegistryClient | 15 + .../nifi-atlassian-nar/pom.xml | 42 ++ .../nifi-atlassian-bundle/pom.xml | 32 + .../git/AbstractGitFlowRegistryClient.java | 3 +- nifi-extension-bundles/pom.xml | 1 + 9 files changed, 932 insertions(+), 1 deletion(-) create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketAuthenticationType.java create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/resources/META-INF/services/org.apache.nifi.registry.flow.FlowRegistryClient create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml create mode 100644 nifi-extension-bundles/nifi-atlassian-bundle/pom.xml diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml new file mode 100644 index 000000000000..439693e7e346 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + org.apache.nifi + nifi-atlassian-bundle + 2.1.0-SNAPSHOT + + nifi-atlassian-extensions + jar + + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-git-flow-registry + 2.1.0-SNAPSHOT + + + org.apache.nifi + nifi-oauth2-provider-api + + + org.apache.nifi + nifi-web-client-provider-api + + + org.apache.nifi + nifi-web-client-api + + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketAuthenticationType.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketAuthenticationType.java new file mode 100644 index 000000000000..fcb0369d04f0 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketAuthenticationType.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.atlassian.bitbucket; + +import org.apache.nifi.components.DescribedValue; + +public enum BitBucketAuthenticationType implements DescribedValue { + BASIC_AUTH("Basic Auth", "Username and App Password"), + ACCESS_TOKEN("Access Token", "Repository, Project or Workspace Token"), + OAUTH2("OAuth 2.0", "OAuth 2.0 with an OAuth Consumer"); + + private final String displayName; + private final String description; + + BitBucketAuthenticationType(final String displayName, final String description) { + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return name(); + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java new file mode 100644 index 000000000000..68b696f29a49 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.atlassian.bitbucket; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.oauth2.OAuth2AccessTokenProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.registry.flow.FlowRegistryClientConfigurationContext; +import org.apache.nifi.registry.flow.FlowRegistryException; +import org.apache.nifi.registry.flow.git.AbstractGitFlowRegistryClient; +import org.apache.nifi.registry.flow.git.client.GitRepositoryClient; +import org.apache.nifi.web.client.provider.api.WebClientServiceProvider; + +import java.util.List; + +@Tags({ "atlassian", "bitbucket", "registry", "flow" }) +@CapabilityDescription("Flow Registry Client that uses the BitBucket REST API to version control flows in a BitBucket Repository.") +public class BitBucketFlowRegistryClient extends AbstractGitFlowRegistryClient { + + static final PropertyDescriptor BITBUCKET_API_URL = new PropertyDescriptor.Builder() + .name("BitBucket API Instance") + .description("The instance of the BitBucket API") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue("api.bitbucket.org") + .required(true) + .build(); + + static final PropertyDescriptor BITBUCKET_API_VERSION = new PropertyDescriptor.Builder() + .name("BitBucket API Version") + .description("The version of the BitBucket API") + .defaultValue("2.0") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .build(); + + static final PropertyDescriptor WORKSPACE_NAME = new PropertyDescriptor.Builder() + .name("Workspace Name") + .description("The name of the workspace that contains the repository to connect to") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .build(); + + static final PropertyDescriptor REPOSITORY_NAME = new PropertyDescriptor.Builder() + .name("Repository Name") + .description("The name of the repository") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .build(); + + static final PropertyDescriptor AUTHENTICATION_TYPE = new PropertyDescriptor.Builder() + .name("Authentication Type") + .description("The type of authentication to use for accessing BitBucket") + .allowableValues(BitBucketAuthenticationType.class) + .defaultValue(BitBucketAuthenticationType.ACCESS_TOKEN) + .required(true) + .build(); + + static final PropertyDescriptor ACCESS_TOKEN = new PropertyDescriptor.Builder() + .name("Access Token") + .description("The access token to use for authentication") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .sensitive(true) + .dependsOn(AUTHENTICATION_TYPE, BitBucketAuthenticationType.ACCESS_TOKEN) + .build(); + + static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() + .name("Username") + .description("The username to use for authentication") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .sensitive(false) + .dependsOn(AUTHENTICATION_TYPE, BitBucketAuthenticationType.BASIC_AUTH) + .build(); + + static final PropertyDescriptor APP_PASSWORD = new PropertyDescriptor.Builder() + .name("App Password") + .description("The App Password to use for authentication") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(true) + .sensitive(true) + .dependsOn(AUTHENTICATION_TYPE, BitBucketAuthenticationType.BASIC_AUTH) + .build(); + + static final PropertyDescriptor OAUTH_TOKEN_PROVIDER = new PropertyDescriptor.Builder() + .name("OAuth2 Access Token Provider") + .description("Service providing OAuth2 Access Tokens for authentication") + .identifiesControllerService(OAuth2AccessTokenProvider.class) + .required(true) + .dependsOn(AUTHENTICATION_TYPE, BitBucketAuthenticationType.OAUTH2) + .build(); + + static final PropertyDescriptor WEBCLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("Web Client Service") + .description("The Web Client Service to use for communicating with BitBucket") + .required(true) + .identifiesControllerService(WebClientServiceProvider.class) + .build(); + + static final List PROPERTY_DESCRIPTORS = List.of( + WEBCLIENT_SERVICE, + BITBUCKET_API_URL, + BITBUCKET_API_VERSION, + WORKSPACE_NAME, + REPOSITORY_NAME, + AUTHENTICATION_TYPE, + ACCESS_TOKEN, + USERNAME, + APP_PASSWORD, + OAUTH_TOKEN_PROVIDER); + + @Override + protected List createPropertyDescriptors() { + return PROPERTY_DESCRIPTORS; + } + + @Override + protected GitRepositoryClient createRepositoryClient(final FlowRegistryClientConfigurationContext context) throws FlowRegistryException { + return BitBucketRepositoryClient.builder() + .clientId(getIdentifier()) + .apiUrl(context.getProperty(BITBUCKET_API_URL).getValue()) + .apiVersion(context.getProperty(BITBUCKET_API_VERSION).getValue()) + .workspace(context.getProperty(WORKSPACE_NAME).getValue()) + .repoName(context.getProperty(REPOSITORY_NAME).getValue()) + .repoPath(context.getProperty(REPOSITORY_PATH).getValue()) + .authenticationType(context.getProperty(AUTHENTICATION_TYPE).asAllowableValue(BitBucketAuthenticationType.class)) + .accessToken(context.getProperty(ACCESS_TOKEN).evaluateAttributeExpressions().getValue()) + .username(context.getProperty(USERNAME).evaluateAttributeExpressions().getValue()) + .appPassword(context.getProperty(APP_PASSWORD).evaluateAttributeExpressions().getValue()) + .oauthService(context.getProperty(OAUTH_TOKEN_PROVIDER).asControllerService(OAuth2AccessTokenProvider.class)) + .webClient(context.getProperty(WEBCLIENT_SERVICE).asControllerService(WebClientServiceProvider.class)) + .build(); + } + + @Override + public boolean isStorageLocationApplicable(FlowRegistryClientConfigurationContext context, String location) { + // TODO Auto-generated method stub + return false; + } + + @Override + protected String getStorageLocation(GitRepositoryClient repositoryClient) { + // TODO Auto-generated method stub + return null; + } +} diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java new file mode 100644 index 000000000000..3561c5b0b7e4 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java @@ -0,0 +1,576 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.atlassian.bitbucket; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.apache.nifi.oauth2.OAuth2AccessTokenProvider; +import org.apache.nifi.registry.flow.FlowRegistryException; +import org.apache.nifi.registry.flow.git.client.GitCommit; +import org.apache.nifi.registry.flow.git.client.GitCreateContentRequest; +import org.apache.nifi.registry.flow.git.client.GitRepositoryClient; +import org.apache.nifi.web.client.api.HttpResponseEntity; +import org.apache.nifi.web.client.api.HttpUriBuilder; +import org.apache.nifi.web.client.api.StandardHttpContentType; +import org.apache.nifi.web.client.api.StandardMultipartFormDataStreamBuilder; +import org.apache.nifi.web.client.provider.api.WebClientServiceProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +/** + * Implementation of {@link GitRepositoryClient} for BitBucket. + */ +public class BitBucketRepositoryClient implements GitRepositoryClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(BitBucketRepositoryClient.class); + + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private static final String BASIC = "Basic"; + private static final String BEARER = "Bearer"; + + private final ObjectMapper objectMapper = JsonMapper.builder().build(); + + private final String apiUrl; + private final String apiVersion; + private final String clientId; + private final String workspace; + private final String repoName; + private final String repoPath; + private WebClientServiceProvider webClient; + private BitBucketToken authToken; + + private final boolean canRead; + private final boolean canWrite; + + private BitBucketRepositoryClient(final Builder builder) throws FlowRegistryException { + webClient = Objects.requireNonNull(builder.webClient, "Web Client is required"); + workspace = Objects.requireNonNull(builder.workspace, "Workspace is required"); + repoName = Objects.requireNonNull(builder.repoName, "Repository Name is required"); + + apiUrl = Objects.requireNonNull(builder.apiUrl, "API Instance is required"); + apiVersion = Objects.requireNonNull(builder.apiVersion, "API Version is required"); + + final BitBucketAuthenticationType authenticationType = Objects.requireNonNull(builder.authenticationType, "Authentication type is required"); + + switch (authenticationType) { + case ACCESS_TOKEN -> { + Objects.requireNonNull(builder.accessToken, "Access Token is required"); + authToken = new AccessToken(builder.accessToken); + } + case BASIC_AUTH -> { + Objects.requireNonNull(builder.username, "Username is required"); + Objects.requireNonNull(builder.appPassword, "App Password URL is required"); + authToken = new BasicAuthToken(builder.username, builder.appPassword); + } + case OAUTH2 -> { + Objects.requireNonNull(builder.oauthService, "OAuth 2.0 Token Provider is required"); + authToken = new OAuthToken(builder.oauthService); + } + } + + clientId = Objects.requireNonNull(builder.clientId, "Client ID is required"); + repoPath = builder.repoPath; + + final String permission = checkRepoPermissions(); + + switch (permission) { + case "admin", "write" -> { + canRead = true; + canWrite = true; + } + case "read" -> { + canRead = true; + canWrite = false; + } + case "none" -> { + canRead = false; + canWrite = false; + } + default -> { + canRead = false; + canWrite = false; + } + } + + LOGGER.info("Created {} for clientId = [{}], repository [{}]", getClass().getSimpleName(), clientId, repoName); + } + + @Override + public boolean hasReadPermission() { + return canRead; + } + + @Override + public boolean hasWritePermission() { + return canWrite; + } + + @Override + public Set getBranches() throws FlowRegistryException { + LOGGER.debug("Getting branches for repository [{}]", repoName); + + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/refs/branches + URI uri = getUriBuilder().addPathSegment("refs").addPathSegment("branches").build(); + HttpResponseEntity response = this.webClient.getWebClientService().get().uri(uri).header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()).retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_OK) { + throw new FlowRegistryException(String.format("Error while listing branches for repository [%s]: %s", repoName, getErrorMessage(response))); + } + + JsonNode jsonResponse; + try { + jsonResponse = this.objectMapper.readTree(response.body()); + } catch (IOException e) { + throw new FlowRegistryException("Could not parse response from BitBucket API", e); + } + Iterator branches = jsonResponse.get("values").elements(); + + Set result = new HashSet(); + while (branches.hasNext()) { + JsonNode branch = branches.next(); + result.add(branch.get("name").asText()); + } + + return result; + } + + @Override + public Set getTopLevelDirectoryNames(final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(""); + LOGGER.debug("Getting top-level directories for path [{}] on branch [{}] in repository [{}]", resolvedPath, branch, repoName); + + final Iterator files = getFiles(branch, resolvedPath); + + final Set result = new HashSet(); + while (files.hasNext()) { + JsonNode file = files.next(); + if (file.get("type").asText().equals("commit_directory")) { + final Path fullPath = Paths.get(file.get("path").asText()); + result.add(fullPath.getFileName().toString()); + } + } + + return result; + } + + @Override + public Set getFileNames(final String directory, final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(directory); + LOGGER.debug("Getting filenames for path [{}] on branch [{}] in repository [{}]", resolvedPath, branch, repoName); + + final Iterator files = getFiles(branch, resolvedPath); + + final Set result = new HashSet(); + while (files.hasNext()) { + JsonNode file = files.next(); + if (file.get("type").asText().equals("commit_file")) { + final Path fullPath = Paths.get(file.get("path").asText()); + result.add(fullPath.getFileName().toString()); + } + } + + return result; + } + + @Override + public List getCommits(final String path, final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(path); + LOGGER.debug("Getting commits for path [{}] on branch [{}] in repository [{}]", resolvedPath, branch, repoName); + + Iterator commits = getListCommits(branch, resolvedPath); + + final List result = new ArrayList(); + while (commits.hasNext()) { + JsonNode commit = commits.next(); + result.add(toGitCommit(commit)); + } + + return result; + } + + @Override + public InputStream getContentFromBranch(final String path, final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(path); + LOGGER.debug("Getting content for path [{}] on branch [{}] in repository [{}]", resolvedPath, branch, repoName); + final Optional lastCommit = getLatestCommit(branch, resolvedPath); + + if (lastCommit.isEmpty()) { + throw new FlowRegistryException(String.format("Could not find committed files at %s on branch %s response from BitBucket API", resolvedPath, branch)); + } + return getContentFromCommit(path, lastCommit.get()); + } + + @Override + public InputStream getContentFromCommit(final String path, final String commitSha) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(path); + LOGGER.debug("Getting content for path [{}] from commit [{}] in repository [{}]", resolvedPath, commitSha, repoName); + + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/src/{commit}/{path} + final URI uri = getUriBuilder().addPathSegment("src").addPathSegment(commitSha).addPathSegment(resolvedPath).build(); + final HttpResponseEntity response = this.webClient.getWebClientService().get().uri(uri).header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()).retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_OK) { + throw new FlowRegistryException( + String.format("Error while retrieving content for repository [%s] at path %s: %s", repoName, resolvedPath, getErrorMessage(response))); + } + + return response.body(); + } + + @Override + public Optional getContentSha(final String path, final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(path); + LOGGER.debug("Getting content SHA for path [{}] on branch [{}] in repository [{}]", resolvedPath, branch, repoName); + return getLatestCommit(branch, resolvedPath); + } + + @Override + public String createContent(final GitCreateContentRequest request) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(request.getPath()); + final String branch = request.getBranch(); + LOGGER.debug("Creating content at path [{}] on branch [{}] in repository [{}] ", resolvedPath, branch, repoName); + + final StandardMultipartFormDataStreamBuilder multipartBuilder = new StandardMultipartFormDataStreamBuilder(); + multipartBuilder.addPart(resolvedPath, StandardHttpContentType.APPLICATION_JSON, request.getContent().getBytes(StandardCharsets.UTF_8)); + multipartBuilder.addPart("message", StandardHttpContentType.TEXT_PLAIN, request.getMessage().getBytes(StandardCharsets.UTF_8)); + multipartBuilder.addPart("branch", StandardHttpContentType.TEXT_PLAIN, branch.getBytes(StandardCharsets.UTF_8)); + + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/src + final URI uri = getUriBuilder().addPathSegment("src").build(); + final HttpResponseEntity response = this.webClient.getWebClientService() + .post() + .uri(uri) + .body(multipartBuilder.build(), OptionalLong.empty()) + .header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()) + .header(CONTENT_TYPE_HEADER, multipartBuilder.getHttpContentType().getContentType()) + .retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_CREATED) { + throw new FlowRegistryException( + String.format("Error while committing content for repository [%s] on branch %s at path %s: %s", repoName, branch, resolvedPath, getErrorMessage(response))); + } + + final Optional lastCommit = getLatestCommit(branch, resolvedPath); + + if (lastCommit.isEmpty()) { + throw new FlowRegistryException(String.format("Could not find commit for the file %s we just tried to commit on branch %s", resolvedPath, branch)); + } + + return lastCommit.get(); + } + + @Override + public InputStream deleteContent(final String filePath, final String commitMessage, final String branch) throws FlowRegistryException { + final String resolvedPath = getResolvedPath(filePath); + LOGGER.debug("Deleting content at path [{}] on branch [{}] in repository [{}] ", resolvedPath, branch, repoName); + + final InputStream fileToBeDeleted = getContentFromBranch(filePath, branch); + + final StandardMultipartFormDataStreamBuilder multipartBuilder = new StandardMultipartFormDataStreamBuilder(); + multipartBuilder.addPart("files", StandardHttpContentType.TEXT_PLAIN, resolvedPath.getBytes(StandardCharsets.UTF_8)); + multipartBuilder.addPart("message", StandardHttpContentType.TEXT_PLAIN, commitMessage.getBytes(StandardCharsets.UTF_8)); + multipartBuilder.addPart("branch", StandardHttpContentType.TEXT_PLAIN, branch.getBytes(StandardCharsets.UTF_8)); + + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/src + final URI uri = getUriBuilder().addPathSegment("src").build(); + final HttpResponseEntity response = this.webClient.getWebClientService() + .post() + .uri(uri) + .body(multipartBuilder.build(), OptionalLong.empty()) + .header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()) + .header(CONTENT_TYPE_HEADER, multipartBuilder.getHttpContentType().getContentType()) + .retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_CREATED) { + throw new FlowRegistryException( + String.format("Error while deleting content for repository [%s] on branch %s at path %s: %s", repoName, branch, resolvedPath, getErrorMessage(response))); + } + + return fileToBeDeleted; + } + + private Iterator getFiles(final String branch, final String resolvedPath) throws FlowRegistryException { + final Optional lastCommit = getLatestCommit(branch, resolvedPath); + + if (lastCommit.isEmpty()) { + throw new FlowRegistryException(String.format("Could not find committed files at %s on branch %s response from BitBucket API", resolvedPath, branch)); + } + + // retrieve source data + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/src/{commit}/{path} + final URI uri = getUriBuilder().addPathSegment("src").addPathSegment(lastCommit.get()).addPathSegment(resolvedPath).build(); + final HttpResponseEntity response = this.webClient.getWebClientService().get().uri(uri).header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()).retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_OK) { + throw new FlowRegistryException( + String.format("Error while listing content for repository [%s] on branch %s at path %s: %s", repoName, branch, resolvedPath, getErrorMessage(response))); + } + + final JsonNode jsonResponse; + try { + jsonResponse = this.objectMapper.readTree(response.body()); + } catch (IOException e) { + throw new FlowRegistryException("Could not parse response from BitBucket API", e); + } + return jsonResponse.get("values").elements(); + } + + private Iterator getListCommits(final String branch, final String path) throws FlowRegistryException { + // retrieve latest commit for that branch + // https://api.bitbucket.org/2.0/repositories/{workspace}/{repoName}/commits/{branch} + final URI uri = getUriBuilder().addPathSegment("commits").addPathSegment(branch).addQueryParameter("path", path).build(); + final HttpResponseEntity response = this.webClient.getWebClientService().get().uri(uri).header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()).retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_OK) { + throw new FlowRegistryException(String.format("Error while listing commits for repository [%s] on branch %s: %s", repoName, branch, getErrorMessage(response))); + } + + final JsonNode jsonResponse; + try { + jsonResponse = this.objectMapper.readTree(response.body()); + } catch (IOException e) { + throw new FlowRegistryException("Could not parse response from BitBucket API", e); + } + return jsonResponse.get("values").elements(); + } + + private Optional getLatestCommit(final String branch, final String path) throws FlowRegistryException { + Iterator commits = getListCommits(branch, path); + if (commits.hasNext()) { + return Optional.of(commits.next().get("hash").asText()); + } else { + return Optional.empty(); + } + } + + private String checkRepoPermissions() throws FlowRegistryException { + LOGGER.debug("Retrieving information about current user"); + + // 'https://api.bitbucket.org/2.0/user/permissions/repositories?q=repository.name="{repoName}" + URI uri = this.webClient.getHttpUriBuilder() + .scheme("https") + .host(apiUrl) + .addPathSegment(apiVersion) + .addPathSegment("user") + .addPathSegment("permissions") + .addPathSegment("repositories") + .addQueryParameter("q", "repository.name=\"" + repoName + "\"") + .build(); + HttpResponseEntity response = this.webClient.getWebClientService().get().uri(uri).header(AUTHORIZATION_HEADER, authToken.getAuthzHeaderValue()).retrieve(); + + if (response.statusCode() != HttpURLConnection.HTTP_OK) { + throw new FlowRegistryException(String.format("Error while retrieving permission metadata for specified repo - %s", getErrorMessage(response))); + } + + JsonNode jsonResponse; + try { + jsonResponse = this.objectMapper.readTree(response.body()); + } catch (IOException e) { + throw new FlowRegistryException("Could not parse response from BitBucket API", e); + } + Iterator repoPermissions = jsonResponse.get("values").elements(); + + if (repoPermissions.hasNext()) { + return repoPermissions.next().get("permission").asText(); + } else { + return "none"; + } + } + + private GitCommit toGitCommit(final JsonNode commit) { + return new GitCommit( + commit.get("hash").asText(), + commit.get("author").get("raw").asText(), + commit.get("message").asText(), + Instant.parse(commit.get("date").asText())); + } + + private String getErrorMessage(HttpResponseEntity response) throws FlowRegistryException { + final JsonNode jsonResponse; + try { + jsonResponse = this.objectMapper.readTree(response.body()); + } catch (IOException e) { + throw new FlowRegistryException("Could not parse response from BitBucket API", e); + } + return String.format("[%s] - %s", jsonResponse.get("type").asText(), jsonResponse.get("error").get("message").asText()); + } + + private String getResolvedPath(final String path) { + return repoPath == null ? path : repoPath + "/" + path; + } + + private HttpUriBuilder getUriBuilder() { + return this.webClient.getHttpUriBuilder() + .scheme("https") + .host(apiUrl) + .addPathSegment(apiVersion) + .addPathSegment("repositories") + .addPathSegment(workspace) + .addPathSegment(repoName); + } + + private interface BitBucketToken { + T getAuthzHeaderValue(); + } + + private class BasicAuthToken implements BitBucketToken { + private String token; + + public BasicAuthToken(final String username, final String appPassword) { + final String basicCreds = username + ":" + appPassword; + final byte[] basicCredsBytes = basicCreds.getBytes(StandardCharsets.UTF_8); + + final Base64.Encoder encoder = Base64.getEncoder(); + token = encoder.encodeToString(basicCredsBytes); + } + + @Override + public String getAuthzHeaderValue() { + return BASIC + " " + token; + } + } + + private class AccessToken implements BitBucketToken { + private String token; + + public AccessToken(final String token) { + this.token = token; + } + + @Override + public String getAuthzHeaderValue() { + return BEARER + " " + token; + } + } + + private class OAuthToken implements BitBucketToken { + private OAuth2AccessTokenProvider oauthService; + + public OAuthToken(final OAuth2AccessTokenProvider oauthService) { + this.oauthService = oauthService; + } + + @Override + public String getAuthzHeaderValue() { + return BEARER + " " + oauthService.getAccessDetails().getAccessToken(); + } + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String clientId; + private String apiUrl; + private String apiVersion; + private BitBucketAuthenticationType authenticationType; + private String accessToken; + private String username; + private String appPassword; + private OAuth2AccessTokenProvider oauthService; + private WebClientServiceProvider webClient; + private String workspace; + private String repoName; + private String repoPath; + + public Builder clientId(final String clientId) { + this.clientId = clientId; + return this; + } + + public Builder apiUrl(final String apiUrl) { + this.apiUrl = apiUrl; + return this; + } + + public Builder apiVersion(final String apiVersion) { + this.apiVersion = apiVersion; + return this; + } + + public Builder authenticationType(final BitBucketAuthenticationType authenticationType) { + this.authenticationType = authenticationType; + return this; + } + + public Builder accessToken(final String accessToken) { + this.accessToken = accessToken; + return this; + } + + public Builder username(final String username) { + this.username = username; + return this; + } + + public Builder appPassword(final String appPassword) { + this.appPassword = appPassword; + return this; + } + + public Builder oauthService(final OAuth2AccessTokenProvider oauthService) { + this.oauthService = oauthService; + return this; + } + + public Builder webClient(final WebClientServiceProvider webClient) { + this.webClient = webClient; + return this; + } + + public Builder workspace(final String workspace) { + this.workspace = workspace; + return this; + } + + public Builder repoName(final String repoName) { + this.repoName = repoName; + return this; + } + + public Builder repoPath(final String repoPath) { + this.repoPath = repoPath; + return this; + } + + public BitBucketRepositoryClient build() throws FlowRegistryException { + return new BitBucketRepositoryClient(this); + } + } +} diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/resources/META-INF/services/org.apache.nifi.registry.flow.FlowRegistryClient b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/resources/META-INF/services/org.apache.nifi.registry.flow.FlowRegistryClient new file mode 100644 index 000000000000..5022bd4ed678 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/resources/META-INF/services/org.apache.nifi.registry.flow.FlowRegistryClient @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.atlassian.bitbucket.BitBucketFlowRegistryClient \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml new file mode 100644 index 000000000000..53b011a23ceb --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-atlassian-bundle + 2.1.0-SNAPSHOT + + + nifi-atlassian-nar + nar + + + + org.apache.nifi + nifi-atlassian-extensions + 2.1.0-SNAPSHOT + + + org.apache.nifi + nifi-standard-shared-nar + 2.1.0-SNAPSHOT + nar + + + + diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml new file mode 100644 index 000000000000..a4abce75baf2 --- /dev/null +++ b/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + nifi-standard-shared-bom + org.apache.nifi + 2.1.0-SNAPSHOT + ../nifi-standard-shared-bundle/nifi-standard-shared-bom + + + nifi-atlassian-bundle + pom + + + nifi-atlassian-extensions + nifi-atlassian-nar + + diff --git a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java index ad2c25cc61f3..e1e840d08811 100644 --- a/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java +++ b/nifi-extension-bundles/nifi-extension-utils/nifi-git-flow-registry/src/main/java/org/apache/nifi/registry/flow/git/AbstractGitFlowRegistryClient.java @@ -252,6 +252,7 @@ public RegisteredFlow deregisterFlow(final FlowRegistryClientConfigurationContex final String commitMessage = DEREGISTER_FLOW_MESSAGE_FORMAT.formatted(flowLocation.getFlowId()); try (final InputStream deletedSnapshotContent = repositoryClient.deleteContent(filePath, commitMessage, branch)) { final RegisteredFlowSnapshot deletedSnapshot = getSnapshot(deletedSnapshotContent); + populateFlowAndSnapshotMetadata(deletedSnapshot, flowLocation); updateBucketReferences(repositoryClient, deletedSnapshot, flowLocation.getBucketId()); return deletedSnapshot.getFlow(); } @@ -579,9 +580,9 @@ protected synchronized GitRepositoryClient getRepositoryClient(final FlowRegistr if (!clientInitialized.get()) { getLogger().info("Initializing repository client"); repositoryClient = createRepositoryClient(context); - clientInitialized.set(true); initializeDefaultBucket(context); directoryExclusionPattern = Pattern.compile(context.getProperty(DIRECTORY_FILTER_EXCLUDE).getValue()); + clientInitialized.set(true); } return repositoryClient; } diff --git a/nifi-extension-bundles/pom.xml b/nifi-extension-bundles/pom.xml index 2271fa6af25e..a04c8a624032 100755 --- a/nifi-extension-bundles/pom.xml +++ b/nifi-extension-bundles/pom.xml @@ -94,5 +94,6 @@ nifi-protobuf-bundle nifi-github-bundle nifi-gitlab-bundle + nifi-atlassian-bundle From a8e0a42c5ea136c4465ebb68e47f63c7c16e65ef Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 25 Nov 2024 21:14:16 +0100 Subject: [PATCH 2/4] review --- .../bitbucket/BitBucketFlowRegistryClient.java | 10 ++++++---- .../bitbucket/BitBucketRepositoryClient.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java index 68b696f29a49..4f6ab4af38ca 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketFlowRegistryClient.java @@ -126,6 +126,9 @@ public class BitBucketFlowRegistryClient extends AbstractGitFlowRegistryClient { APP_PASSWORD, OAUTH_TOKEN_PROVIDER); + static final String STORAGE_LOCATION_PREFIX = "git@bitbucket.org:"; + static final String STORAGE_LOCATION_FORMAT = STORAGE_LOCATION_PREFIX + "%s/%s.git"; + @Override protected List createPropertyDescriptors() { return PROPERTY_DESCRIPTORS; @@ -151,13 +154,12 @@ protected GitRepositoryClient createRepositoryClient(final FlowRegistryClientCon @Override public boolean isStorageLocationApplicable(FlowRegistryClientConfigurationContext context, String location) { - // TODO Auto-generated method stub - return false; + return location != null && location.startsWith(STORAGE_LOCATION_PREFIX); } @Override protected String getStorageLocation(GitRepositoryClient repositoryClient) { - // TODO Auto-generated method stub - return null; + final BitBucketRepositoryClient gitLabRepositoryClient = (BitBucketRepositoryClient) repositoryClient; + return STORAGE_LOCATION_FORMAT.formatted(gitLabRepositoryClient.getWorkspace(), gitLabRepositoryClient.getRepoName()); } } diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java index 3561c5b0b7e4..d0d619ff85ad 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java @@ -140,6 +140,20 @@ public boolean hasWritePermission() { return canWrite; } + /** + * @return the name of the workspace + */ + public String getWorkspace() { + return workspace; + } + + /** + * @return the name of the repository + */ + public String getRepoName() { + return repoName; + } + @Override public Set getBranches() throws FlowRegistryException { LOGGER.debug("Getting branches for repository [{}]", repoName); From ff28bb8c3f309d7b13590286c7254e86b34500b2 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 6 Jan 2025 18:22:49 +0100 Subject: [PATCH 3/4] Update version to 2.2.0-SNAPSHOT --- .../nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml | 4 ++-- .../nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml | 6 +++--- nifi-extension-bundles/nifi-atlassian-bundle/pom.xml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml index 439693e7e346..10564435ead5 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/pom.xml @@ -17,7 +17,7 @@ org.apache.nifi nifi-atlassian-bundle - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT nifi-atlassian-extensions jar @@ -30,7 +30,7 @@ org.apache.nifi nifi-git-flow-registry - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT org.apache.nifi diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml index 53b011a23ceb..37b7fdcb6a68 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-nar/pom.xml @@ -19,7 +19,7 @@ org.apache.nifi nifi-atlassian-bundle - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT nifi-atlassian-nar @@ -29,12 +29,12 @@ org.apache.nifi nifi-atlassian-extensions - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT org.apache.nifi nifi-standard-shared-nar - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT nar diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml b/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml index a4abce75baf2..35f241c403b2 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml +++ b/nifi-extension-bundles/nifi-atlassian-bundle/pom.xml @@ -18,7 +18,7 @@ nifi-standard-shared-bom org.apache.nifi - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT ../nifi-standard-shared-bundle/nifi-standard-shared-bom From 3cb2e51de609087a521de87d2e5651afef4daf85 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Tue, 7 Jan 2025 22:53:59 +0100 Subject: [PATCH 4/4] checkstyle --- .../atlassian/bitbucket/BitBucketRepositoryClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java index d0d619ff85ad..366c5476d11c 100644 --- a/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java +++ b/nifi-extension-bundles/nifi-atlassian-bundle/nifi-atlassian-extensions/src/main/java/org/apache/nifi/atlassian/bitbucket/BitBucketRepositoryClient.java @@ -174,7 +174,7 @@ public Set getBranches() throws FlowRegistryException { } Iterator branches = jsonResponse.get("values").elements(); - Set result = new HashSet(); + Set result = new HashSet<>(); while (branches.hasNext()) { JsonNode branch = branches.next(); result.add(branch.get("name").asText()); @@ -190,7 +190,7 @@ public Set getTopLevelDirectoryNames(final String branch) throws FlowReg final Iterator files = getFiles(branch, resolvedPath); - final Set result = new HashSet(); + final Set result = new HashSet<>(); while (files.hasNext()) { JsonNode file = files.next(); if (file.get("type").asText().equals("commit_directory")) { @@ -209,7 +209,7 @@ public Set getFileNames(final String directory, final String branch) thr final Iterator files = getFiles(branch, resolvedPath); - final Set result = new HashSet(); + final Set result = new HashSet<>(); while (files.hasNext()) { JsonNode file = files.next(); if (file.get("type").asText().equals("commit_file")) { @@ -228,7 +228,7 @@ public List getCommits(final String path, final String branch) throws Iterator commits = getListCommits(branch, resolvedPath); - final List result = new ArrayList(); + final List result = new ArrayList<>(); while (commits.hasNext()) { JsonNode commit = commits.next(); result.add(toGitCommit(commit));