diff --git a/front50-oss/README.md b/front50-oss/README.md new file mode 100644 index 000000000..c6e7e1b1f --- /dev/null +++ b/front50-oss/README.md @@ -0,0 +1,15 @@ +## Configuring OSS store for front50 + +#### OSS: +```yaml + oss: + enabled: true + endpoint: oss-cn-hangzhou.aliyuncs.com + bucket: + accessKeyId: + accessSecretKey: + versioning: true + readOnlyMode: false +``` + +[Get started with OSS>>](https://www.alibabacloud.com/help/en/object-storage-service/latest/get-started-with-oss) \ No newline at end of file diff --git a/front50-oss/front50-oss.gradle b/front50-oss/front50-oss.gradle new file mode 100644 index 000000000..b39eaaf84 --- /dev/null +++ b/front50-oss/front50-oss.gradle @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":front50-core") + implementation project(":front50-api") + + implementation "org.springframework.boot:spring-boot-starter-web" + implementation "org.springframework.boot:spring-boot-starter-actuator" + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "io.spinnaker.kork:kork-core" + implementation "com.netflix.eureka:eureka-client" + implementation "com.aliyun.oss:aliyun-sdk-oss:3.10.2" + implementation "javax.xml.bind:jaxb-api:2.3.1" + implementation "javax.activation:activation:1.1.1" + implementation "org.glassfish.jaxb:jaxb-runtime:2.3.3" + + + testImplementation project(":front50-test") +} diff --git a/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssClientFactory.java b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssClientFactory.java new file mode 100644 index 000000000..27e7a1cb4 --- /dev/null +++ b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssClientFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.front50.config; + +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; + +public class OssClientFactory { + public static OSS client(OssProperties ossProperties) { + return new OSSClient( + ossProperties.getEndpoint(), + new DefaultCredentialProvider( + ossProperties.getAccessKeyId(), ossProperties.getAccessSecretKey()), + new ClientConfiguration()); + } +} diff --git a/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssConfig.java b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssConfig.java new file mode 100644 index 000000000..6dbbc9925 --- /dev/null +++ b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssConfig.java @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.front50.config; + +import com.aliyun.oss.OSS; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.front50.model.OssStorageService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnProperty("spinnaker.oss.enabled") +@EnableConfigurationProperties(OssProperties.class) +public class OssConfig { + @Bean + public OSS ossClient(OssProperties ossProperties) { + return OssClientFactory.client(ossProperties); + } + + @Bean + public OssStorageService ossStorageService(OSS oss, OssProperties ossProperties) { + OssStorageService storageService = + new OssStorageService( + new ObjectMapper(), + oss, + ossProperties.getBucket(), + ossProperties.getRootFolder(), + ossProperties.getVersioning(), + ossProperties.getReadOnlyMode(), + ossProperties.getMaxKeys()); + storageService.ensureBucketExists(); + return storageService; + } +} diff --git a/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssProperties.java b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssProperties.java new file mode 100644 index 000000000..f3fca31f2 --- /dev/null +++ b/front50-oss/src/main/java/com/netflix/spinnaker/front50/config/OssProperties.java @@ -0,0 +1,97 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.front50.config; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("spinnaker.oss") +public class OssProperties { + private String endpoint; + private String bucket; + private String accessKeyId; + private String accessSecretKey; + private Integer maxKeys = 1000; + private String rootFolder = "front50"; + private Boolean versioning; + private Boolean readOnlyMode; + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getAccessKeyId() { + return accessKeyId; + } + + public void setAccessKeyId(String accessKeyId) { + this.accessKeyId = accessKeyId; + } + + @JsonIgnore + public String getAccessSecretKey() { + return accessSecretKey; + } + + public void setAccessSecretKey(String accessSecretKey) { + this.accessSecretKey = accessSecretKey; + } + + public Integer getMaxKeys() { + return maxKeys; + } + + public void setMaxKeys(Integer maxKeys) { + this.maxKeys = maxKeys; + } + + public String getRootFolder() { + return rootFolder; + } + + public void setRootFolder(String rootFolder) { + this.rootFolder = rootFolder; + } + + public Boolean getVersioning() { + return versioning; + } + + public void setVersioning(Boolean versioning) { + this.versioning = versioning; + } + + public Boolean getReadOnlyMode() { + return readOnlyMode; + } + + public void setReadOnlyMode(Boolean readOnlyMode) { + this.readOnlyMode = readOnlyMode; + } +} diff --git a/front50-oss/src/main/java/com/netflix/spinnaker/front50/model/OssStorageService.java b/front50-oss/src/main/java/com/netflix/spinnaker/front50/model/OssStorageService.java new file mode 100644 index 000000000..b673662da --- /dev/null +++ b/front50-oss/src/main/java/com/netflix/spinnaker/front50/model/OssStorageService.java @@ -0,0 +1,302 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.front50.model; + +import static net.logstash.logback.argument.StructuredArguments.value; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.BucketVersioningConfiguration; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.ListVersionsRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.SetBucketVersioningRequest; +import com.aliyun.oss.model.VersionListing; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.front50.api.model.Timestamped; +import com.netflix.spinnaker.kork.web.exceptions.NotFoundException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; + +@Slf4j +public class OssStorageService implements StorageService { + private final ObjectMapper objectMapper; + private final OSS oss; + private final String bucket; + private final String rootFolder; + private final Boolean versioning; + private final Boolean readOnlyMode; + private final Integer maxKeys; + + public OssStorageService( + ObjectMapper objectMapper, + OSS oss, + String bucket, + String rootFolder, + Boolean versioning, + Boolean readOnlyMode, + Integer maxKeys) { + this.objectMapper = objectMapper; + this.oss = oss; + this.bucket = bucket; + this.rootFolder = rootFolder; + this.versioning = versioning; + this.readOnlyMode = readOnlyMode; + this.maxKeys = maxKeys; + } + + public void ensureBucketExists() { + try { + oss.getBucketInfo(bucket); + } catch (OSSException e) { + if ("NoSuchBucket".equals(e.getErrorCode())) { + log.info("NoSuchBucket create bucket:{}", bucket); + oss.createBucket(bucket); + if (versioning) { + log.info("Enabling versioning of the oss bucket {}", bucket); + BucketVersioningConfiguration configuration = + new BucketVersioningConfiguration().withStatus("Enabled"); + + SetBucketVersioningRequest setBucketVersioningRequest = + new SetBucketVersioningRequest(bucket, configuration); + + oss.setBucketVersioning(setBucketVersioningRequest); + } + } else { + throw e; + } + } + } + + @Override + public boolean supportsVersioning() { + return Boolean.TRUE.equals(versioning); + } + + @Override + public T loadObject(ObjectType objectType, String objectKey) + throws NotFoundException { + try { + OSSObject ossObject = + oss.getObject( + bucket, buildOssKey(objectType.group, objectKey, objectType.defaultMetadataFilename)); + T item = deserialize(ossObject, (Class) objectType.clazz); + item.setLastModified(ossObject.getObjectMetadata().getLastModified().getTime()); + return item; + } catch (OSSException e) { + if ("NoSuchKey".equals(e.getErrorCode())) { + throw new NotFoundException(); + } + log.error("loadObject error,objectType:{}, objectKey:{}", objectType, objectKey, e); + throw e; + } catch (IOException e) { + throw new IllegalStateException("Unable to deserialize object (key: " + objectKey + ")", e); + } + } + + @Override + public void deleteObject(ObjectType objectType, String objectKey) { + checkReadOnly(); + oss.deleteObject( + bucket, buildOssKey(objectType.group, objectKey, objectType.defaultMetadataFilename)); + writeLastModified(objectType.group); + } + + @Override + public void storeObject(ObjectType objectType, String objectKey, T item) { + checkReadOnly(); + try { + byte[] bytes = objectMapper.writeValueAsBytes(item); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(bytes.length); + objectMetadata.setContentMD5( + new String(org.apache.commons.codec.binary.Base64.encodeBase64(DigestUtils.md5(bytes)))); + + oss.putObject( + bucket, + buildOssKey(objectType.group, objectKey, objectType.defaultMetadataFilename), + new ByteArrayInputStream(bytes), + objectMetadata); + writeLastModified(objectType.group); + } catch (JsonProcessingException e) { + throw new IllegalStateException(e); + } + } + + @Override + public Map listObjectKeys(ObjectType objectType) { + long startTime = System.currentTimeMillis(); + String nextMarker = null; + ObjectListing objectListing; + List summaries = new ArrayList<>(maxKeys * 3); + do { + objectListing = + oss.listObjects( + new ListObjectsRequest(bucket).withMarker(nextMarker).withMaxKeys(maxKeys)); + summaries.addAll(objectListing.getObjectSummaries()); + nextMarker = objectListing.getNextMarker(); + } while (objectListing.isTruncated()); + + log.debug( + "Took {}ms to fetch {} object keys for {}", + value("fetchTime", (System.currentTimeMillis() - startTime)), + summaries.size(), + value("type", objectType)); + + return summaries.stream() + .filter(s -> filterObjectSummary(s, objectType.defaultMetadataFilename)) + .collect( + Collectors.toMap( + (s -> buildObjectKey(objectType, s.getKey())), + (s -> s.getLastModified().getTime()))); + } + + @Override + public Collection listObjectVersions( + ObjectType objectType, String objectKey, int maxResults) + throws com.netflix.spinnaker.kork.web.exceptions.NotFoundException { + if (maxResults == 1) { + List results = new ArrayList<>(); + results.add(loadObject(objectType, objectKey)); + return results; + } + + try { + VersionListing versionListing = + oss.listVersions( + new ListVersionsRequest( + bucket, + buildOssKey(objectType.group, objectKey, objectType.defaultMetadataFilename), + null, + null, + null, + maxResults)); + return versionListing.getVersionSummaries().stream() + .map( + versionSummary -> { + try { + OSSObject ossObject = + oss.getObject( + new GetObjectRequest( + bucket, + buildOssKey( + objectType.group, objectKey, objectType.defaultMetadataFilename), + versionSummary.getVersionId())); + T item = deserialize(ossObject, (Class) objectType.clazz); + item.setLastModified(ossObject.getObjectMetadata().getLastModified().getTime()); + return item; + } catch (IOException e) { + throw new IllegalStateException(e); + } + }) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("", e); + throw e; + } + } + + @Override + public long getLastModified(ObjectType objectType) { + try { + Map lastModified = + objectMapper.readValue( + oss.getObject( + bucket, + buildTypedFolder(rootFolder, objectType.group) + "/last-modified.json") + .getObjectContent(), + Map.class); + + return lastModified.get("lastModified"); + } catch (Exception e) { + return 0L; + } + } + + private void checkReadOnly() { + if (readOnlyMode) { + throw new RuntimeException("Cannot perform write operation, in read-only mode"); + } + } + + private void writeLastModified(String group) { + checkReadOnly(); + try { + byte[] bytes = + objectMapper.writeValueAsBytes( + Collections.singletonMap("lastModified", System.currentTimeMillis())); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(bytes.length); + objectMetadata.setContentMD5( + new String(org.apache.commons.codec.binary.Base64.encodeBase64(DigestUtils.md5(bytes)))); + oss.putObject( + bucket, + buildTypedFolder(rootFolder, group) + "/last-modified.json", + new ByteArrayInputStream(bytes), + objectMetadata); + } catch (JsonProcessingException e) { + throw new IllegalStateException(e); + } + } + + private boolean filterObjectSummary(OSSObjectSummary objectSummary, String metadataFilename) { + return objectSummary.getKey().endsWith(metadataFilename); + } + + private T deserialize(OSSObject ossObject, Class clazz) + throws IOException { + return objectMapper.readValue(ossObject.getObjectContent(), clazz); + } + + private String buildOssKey(String group, String objectKey, String metadataFilename) { + if (objectKey.endsWith(metadataFilename)) { + return objectKey; + } + + return (buildTypedFolder(rootFolder, group) + + "/" + + objectKey.toLowerCase() + + "/" + + metadataFilename) + .replace("//", "/"); + } + + private String buildObjectKey(ObjectType objectType, String s3Key) { + return s3Key + .replaceAll(buildTypedFolder(rootFolder, objectType.group) + "/", "") + .replaceAll("/" + objectType.defaultMetadataFilename, ""); + } + + private static String buildTypedFolder(String rootFolder, String type) { + return (rootFolder + "/" + type).replaceAll("//", "/"); + } +} diff --git a/front50-oss/src/test/java/com/netflix/spinnaker/front50/model/OssStorageServiceTest.java b/front50-oss/src/test/java/com/netflix/spinnaker/front50/model/OssStorageServiceTest.java new file mode 100644 index 000000000..8d2437d28 --- /dev/null +++ b/front50-oss/src/test/java/com/netflix/spinnaker/front50/model/OssStorageServiceTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2022 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.front50.model; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.Bucket; +import com.aliyun.oss.model.BucketInfo; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.front50.model.application.Application; +import java.io.ByteArrayInputStream; +import java.util.Date; +import org.junit.Test; + +public class OssStorageServiceTest { + String bucket = "test-bucket-112121212"; + OSS oss = mock(OSS.class); + ObjectMapper objectMapper = new ObjectMapper(); + OssStorageService storageService = + new OssStorageService(objectMapper, oss, bucket, "/root", true, false, 1000); + + @Test + public void test_ensureBucketExists() { + when(oss.getBucketInfo(eq(bucket))).thenReturn(new BucketInfo()); + storageService.ensureBucketExists(); + verify(oss, times(0)).createBucket(eq(bucket)); + } + + @Test + public void test_ensureBucketExists_noSuchBucket() { + when(oss.getBucketInfo(eq(bucket))) + .thenThrow(new OSSException("", "NoSuchBucket", "", null, null, null, null)); + when(oss.createBucket(eq(bucket))).thenReturn(new Bucket()); + doNothing().when(oss).setBucketVersioning(any()); + storageService.ensureBucketExists(); + verify(oss, times(1)).createBucket(eq(bucket)); + verify(oss, times(1)).setBucketVersioning(any()); + } + + @Test(expected = OSSException.class) + public void test_ensureBucketExists_throwexception() { + when(oss.getBucketInfo(eq(bucket))) + .thenThrow(new OSSException("", "400", "", null, null, null, null)); + storageService.ensureBucketExists(); + } + + @Test + public void test_loadObject() throws JsonProcessingException { + Date curTime = new Date(); + + Application application = new Application(); + application.setName("test"); + application.setDescription("desc"); + application.setCreatedAt(curTime.getTime()); + + OSSObject object = new OSSObject(); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setLastModified(curTime); + object.setObjectMetadata(metadata); + object.setObjectContent(new ByteArrayInputStream(objectMapper.writeValueAsBytes(application))); + + when(oss.getObject(eq(bucket), anyString())).thenReturn(object); + + Application load = storageService.loadObject(ObjectType.APPLICATION, application.getName()); + assertNotNull(load); + assertEquals(load.getLastModified().longValue(), metadata.getLastModified().getTime()); + assertEquals(load.getName(), application.getName()); + assertEquals(load.getDescription(), load.getDescription()); + } +} diff --git a/gradle.properties b/gradle.properties index 1c16c60f7..6868ac4f9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ fiatVersion=1.30.0 -includeProviders=azure,gcs,oracle,redis,s3,swift,sql +includeProviders=azure,gcs,oracle,redis,s3,swift,sql,oss korkVersion=7.142.0 org.gradle.parallel=true spinnakerGradleVersion=8.23.0 diff --git a/settings.gradle b/settings.gradle index c3099599c..2c978884b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -41,7 +41,8 @@ include 'front50-web', 'front50-azure', 'front50-swift', 'front50-oracle', - 'front50-bom' + 'front50-bom', + 'front50-oss' def setBuildFile(project) { project.buildFileName = "${project.name}.gradle"