Merge branch 'extension-artifacts-build'

.github/workflows/ci.yml

@@ -20,6 +20,11 @@ jobs:
       with:
         path: ~/.m2
         key: lucee-script-runner-maven-cache
+    - name: Cache Test Artifacts
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/test/artifacts
+        key: test-artifacts
     - name: Cache Lucee files
       uses: actions/cache@v4
       with:
@@ -50,10 +55,22 @@ jobs:
         testLabels: data-provider
         testAdditional: ${{ github.workspace }}/tests
         testSevices: mysql
-
+    - name: Run Lucee Test Suite (testLabels="data-provider-integration")
+      uses: lucee/script-runner@main
+      continue-on-error: true
+      with:
+        webroot: ${{ github.workspace }}/lucee/test
+        execute: /bootstrap-tests.cfm
+        luceeVersion: ${{ env.luceeVersion }}
+        #luceeVersionQuery: ${{ env.luceeVersionQuery }}
+        extensionDir: ${{ github.workspace }}/
+      env:
+        testLabels: data-provider-integration
+        testAdditional: ${{ github.workspace }}/tests
+        testSevices: mysql
 
   build-and-push:
-    name: Docker build, publish and deploy
+    name: Docker build & publish
     runs-on: ubuntu-latest
     needs: [javabuild]
     if: "github.ref == 'refs/heads/master' && github.event_name == 'push'"
@@ -79,8 +96,3 @@ jobs:
       run: |
         docker buildx build --platform linux/amd64,linux/arm64 --build-arg SENTRY_ENV=UAT --build-arg SENTRY_DSN=:${{ env.SENTRY_DSN }} -t markdrew/lucee-downloads:latest -t markdrew/lucee-downloads:${{ github.sha }} -f devops/Dockerfile.download . --push
         docker buildx build --platform linux/amd64,linux/arm64 --build-arg SENTRY_ENV=UAT --build-arg SENTRY_DSN=:${{ env.SENTRY_DSN }} -t markdrew/lucee-update:latest -t markdrew/lucee-update:${{ github.sha }} -f devops/Dockerfile.update . --push
-
-    - name: Trigger deployments
-      run: |
-        curl -XPOST -u ${{ secrets.DEPLOY_API_TOKEN }}:${{ secrets.DEPLOY_API_TOKEN }} "${{ secrets.DEPLOY_API_ENDPOINT }}/download/"
-        curl -XPOST -u ${{ secrets.DEPLOY_API_TOKEN }}:${{ secrets.DEPLOY_API_TOKEN }} "${{ secrets.DEPLOY_API_ENDPOINT }}/update/"

.github/workflows/deploydownload.yml

@@ -0,0 +1,11 @@
+name: Deploy download.lucee.org
+on:
+  - workflow_dispatch
+jobs:
+  deploy-to-download:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+       - name: Trigger deployments
+      run: |
+        curl -XPOST -u ${{ secrets.DEPLOY_API_TOKEN }}:${{ secrets.DEPLOY_API_TOKEN }} "${{ secrets.DEPLOY_API_ENDPOINT }}/download/"

.github/workflows/deployupdate.yml

@@ -0,0 +1,11 @@
+name: Deploy update.lucee.org
+on:
+  - workflow_dispatch
+jobs:
+  deploy-to-update:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+       - name: Trigger deployments
+      run: |
+        curl -XPOST -u ${{ secrets.DEPLOY_API_TOKEN }}:${{ secrets.DEPLOY_API_TOKEN }} "${{ secrets.DEPLOY_API_ENDPOINT }}/update/"

.gitignore

@@ -24,3 +24,4 @@ update/WEB-INF
 /download/debug.cfm
 update/log-maven-download.log
 update/missing-bundles.txt
+/tests/artifacts

README.md

@@ -1,3 +1,7 @@
-# lucee-data-provider
+# Lucee "Data provider"
 
-Home of http://stable.lucee.org/download/
+This repo contains the application code for:
+
+* https://download.lucee.org
+* https://update.lucee.org
+* https://extension.lucee.org

apps/updateserver/Application.cfc

@@ -0,0 +1,54 @@
+component {
+
+	this.name            = "lucee-provider";
+	this.s3.accessKeyId  = server.system.environment.S3_EXTENSION_ACCESS_KEY_ID;
+	this.s3.awsSecretKey = server.system.environment.S3_EXTENSION_SECRET_KEY;
+	this.allowReload     = IsBoolean( server.system.environment.ALLOW_RELOAD ?: "" ) && server.system.environment.ALLOW_RELOAD;
+
+	function onApplicationStart() {
+		_loadServices();
+	}
+
+	function onRequestStart() output=true {
+		if ( this.allowReload && StructKeyExists( url, "fwreinit" ) ) {
+			_loadServices();
+		}
+
+		var allowedPaths = [ "rest", "healthcheck" ];
+		var requestedPath = ListFirst( Trim( cgi.script_name ), "/" );
+
+		if ( !ArrayFindNoCase( allowedPaths, requestedPath ) ) {
+			content reset=true;
+			header statuscode=404;
+			echo( "not found" );
+			abort;
+		}
+	}
+
+	function _loadServices() {
+		var extS3Root    = server.system.environment.S3_EXTENSIONS_ROOT    ?: "s3:///extension-downloads/";
+		var extCdnUrl    = server.system.environment.S3_EXTENSIONS_CDN_URL ?: "https://ext.lucee.org/";
+		var bundleS3Root = server.system.environment.S3_BUNDLES_ROOT       ?: "s3:///bundle-download/";
+		var bundleCdnUrl = server.system.environment.S3_BUNDLES_CDN_URL    ?: "https://bundle.lucee.org/";
+
+		var extMetaReader = new services.ExtensionMetadataReader(
+			s3root = extS3Root
+		);
+		var bundleDownloadService = new services.BundleDownloadService(
+			  extensionsCdnUrl    = extCdnUrl
+			, bundleS3Root        = bundleS3Root
+			, bundleCdnUrl        = bundleCdnUrl
+			, extensionMetaReader = extMetaReader
+			, mavenMatcher        = new services.legacy.MavenMatcher()
+		);
+
+		extMetaReader.setBundleDownloadservice( bundleDownloadService );
+		extMetaReader.loadMeta();
+
+		application.extensionsCdnUrl      = extCdnUrl;
+		application.extensionsS3Root      = extS3Root;
+		application.extMetaReader         = extMetaReader;
+		application.bundleDownloadService = bundleDownloadService;
+	}
+
+}

apps/updateserver/rest/ExtensionProvider.cfc

@@ -0,0 +1,161 @@
+/**
+ * @restpath /extension/provider
+ * @rest     true
+ */
+component {
+
+	variables.metaReader = application.extMetaReader;
+	variables.cdnURL     = application.extensionsCdnUrl;
+
+	/**
+	 * @httpmethod GET
+	 * @restPath   info
+	 */
+	remote struct function getInfo(
+		  required boolean withLogo = true      restargsource="url"
+		, required string  type     = 'release' restargsource="url"
+		, required boolean flush    = false     restargsource="url"
+	) {
+		var hostName = cgi.HTTP_HOST;
+		var retVal   = { meta={
+			  title       = "Lucee Association Switzerland Extension Store (#hostName#)"
+			, description = ""
+			, image       = "https://#hostName#/extension/extension-provider.png"
+			, url         = "https://#hostName#"
+			, mode        = "production"
+		} };
+
+		try {
+			retVal.extensions = metaReader.list(
+				  type     = ReFindNoCase( "^beta\.", hostName ) ? "abc" : arguments.type
+				, flush    = arguments.flush
+				, withLogo = arguments.withLogo
+			);
+		} catch( e ) {
+			return e;
+		}
+
+		return retVal;
+	}
+
+	/**
+	 * @httpmethod GET
+	 * @restPath   info/{id}
+	 */
+	remote  function getInfoDetail(
+		  required string  id               restargsource="Path"
+		, required boolean withLogo = true  restargsource="url"
+		,          string  version  = ""    restargsource="url"
+		,          boolean flush    = false restargsource="url"
+	) {
+		var ext = metaReader.getExtensionDetail(
+			  id       = arguments.id
+			, withLogo = arguments.withLogo
+			, version  = arguments.version
+			, flush    = arguments.flush
+		);
+
+		return ext;
+
+		// TODO + some logging + ensure 404 when not found (old logic below)
+
+		if ( StructCount( ext ) ) {
+			return ext;
+		}
+
+		var msg = "There is no extension at this provider with id [#EncodeForHtml( arguments.id )#]";
+		if ( Len( arguments.version ) ) {
+			 msg &= " in version [#EncodeForHtml( arguments.version )#].";
+		} else {
+			msg &= ".";
+		}
+		SystemOutput( msg, true, true );
+		content type="text/plain";
+		header statuscode=404 statustext=msg;
+		echo(msg); // otherwise this creates a stack trace for forgebox stuff
+		abort;
+	}
+
+
+	/**
+	 * provides trial versions ??!
+	 *
+	 * @httpmethod GET
+	 * @restPath   full/{id}
+	 */
+	remote function getFull(
+		  required string  id              restargsource="Path"
+		,          string  version = ""    restargsource="url"
+		,          boolean flush   = false restargsource="url"
+	) {
+
+		var ext = metaReader.getExtensionDetail(
+			  id       = arguments.id
+			, version  = arguments.version
+			, flush    = arguments.flush
+			, withLogo = false
+		);
+
+		if ( StructCount( ext ) ) {
+			header statuscode="302" statustext="Found";
+			header name="Location" value=variables.cdnURL & ext.filename;
+		}
+
+		header statuscode="404" statustext="Not Found";
+	}
+
+	/**
+	 * provides trial versions (not sure what this is - has same logic as /full/{id})
+	 *
+	 * @httpmethod GET
+	 * @restPath   trial/{id}
+	 */
+	remote function getTrial(
+		  required string  id              restargsource="Path"
+		,          string  version = ""    restargsource="url"
+		,          boolean flush   = false restargsource="url"
+	) {
+
+		var ext = metaReader.getExtensionDetail(
+			  id       = arguments.id
+			, version  = arguments.version
+			, flush    = arguments.flush
+			, withLogo = false
+		);
+
+		if ( StructCount( ext ) ) {
+			header statuscode="302" statustext="Found";
+			header name="Location" value=variables.cdnURL & ext.filename;
+		}
+
+		header statuscode="404" statustext="Not Found";
+	}
+
+
+	/**
+	 * Would be good to get rid of this?
+	 *
+	 * @httpmethod GET
+	 * @restPath   updateCache
+	 */
+	remote struct function updateCache(){
+		try {
+			return {
+				  meta       = {}
+				, extensions = metaReader.loadMeta()
+			}
+		} catch( any e ) {
+			return e;
+		}
+	}
+
+	/**
+	 * Would be good to get rid of this?
+	 *
+	 * @httpmethod GET
+	 * @restPath   reset
+	 */
+	remote function reset() {
+		metaReader.loadMeta();
+	}
+}