Create a CICD

US-GHG-Center · Dec 19, 2023 · 8548da9 · 8548da9
1 parent 500111b
commit 8548da9
Show file tree

Hide file tree

Showing 6 changed files with 178 additions and 7 deletions.
diff --git a/.env b/.env
@@ -1,2 +1,2 @@
 baseurl = https://earth.gov
-subpath = /ghgcenter
+subpath = /ghgcenter
diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml
@@ -0,0 +1,55 @@
+name: Scheduled URL Access Check
+
+# Controls when the action will run.
+on:
+  push:
+  # schedule:
+  #   # * is a special character in YAML so you have to quote this string
+  #   - cron:  '30 * * * *'
+  # # Allows you to run this workflow manually from the Actions tab
+  # workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+jobs:
+  link_check_job:
+    runs-on: ubuntu-latest
+    name: Check all sites
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js v20.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20.x'
+      - run: npm ci
+      - run: npm test
+
+  # Single deploy job since we're just deploying
+  deploy:
+    needs: link_check_job
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          # Upload index.html
+          path: './index.html'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
diff --git a/index.html b/index.html
@@ -0,0 +1,15 @@
+
+  <!DOCTYPE html>
+  <html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>OK - Status 200</title>
+  </head>
+  <body>
+
+    <h1>OK - Status 200</h1>
+    <p>No errors found.</p>
+
+  </body>
+  </html>
diff --git a/link_checker_test/index.js b/link_checker_test/index.js
@@ -5,8 +5,10 @@ import rehypeParse from 'rehype-parse';
 import rehypeStringify from 'rehype-stringify';
 import { chromium } from 'playwright';
 import dotenv from 'dotenv'
+import crypto from 'crypto';
+import { Agent } from "undici";
 
-
+import fs from 'fs';
 
 const crawled = [];
 let discoveredLinks = [];
@@ -18,6 +20,73 @@ let failed_urls = []
 dotenv.config({ path: '.env' })
 const baseUrl = process.env.baseurl;
 const startPath = process.env.subpath;
+
+
+const fetch_url = (url) => {
+  return fetch(url, {
+    // @ts-ignore
+    dispatcher: new Agent({
+      connect: {
+        rejectUnauthorized: false,
+        secureOptions: crypto.constants.SSL_OP_LEGACY_SERVER_CONNECT
+      }
+    })
+  })
+}
+const index_response = (title, body) => {
+  return `
+  <!DOCTYPE html>
+  <html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>${title}</title>
+  </head>
+  <body>
+  ${body}
+  </body>
+  </html>
+`;
+
+}
+
+const generate_index_html = (url_errors) => {
+  if (url_errors.length > 0) {
+    // Create error response with status 500 and list of URLs
+    const bodyResponse = `
+
+    <h1>Error 500 - Server Error</h1>
+    <p>List of URLs with errors:</p>
+    <ul>
+      ${url_errors.map(url => `<li>${url}</li>`).join('')}
+    </ul>
+`;
+
+    const errorResponse = index_response("Error 500 - Server Error", bodyResponse)
+
+
+    // Write the error response to index.html
+    fs.writeFileSync('index.html', errorResponse);
+
+    // Return the error status
+    return { status: 500, message: 'Error 500 - Server Error. Check index.html for details.' };
+  } else {
+    // Create OK response with status 200
+    const OkbodyResponse = `
+    <h1>OK - Status 200</h1>
+    <p>No errors found.</p>
+`;
+    const okResponse = index_response("OK - Status 200", OkbodyResponse)
+
+
+    // Write the OK response to index.html
+    fs.writeFileSync('index.html', okResponse);
+
+    // Return the OK status
+    return { status: 200, message: 'OK - Status 200. Check index.html for details.' };
+  }
+};
+
 test('Crawl for bad URIs', async () => {
   // Launch the browser
   const browser = await chromium.launch();
@@ -52,10 +121,10 @@ test('Crawl for bad URIs', async () => {
 
       if (!crawled.includes(links[i]) && links[i].indexOf(baseUrl + startPath) == 0) {
 
-        fetch(links[i])
+        fetch_url(links[i])
         .then((response) => {
 
-                expect(response.status, `Expected a 200 OK response for image ${links[i]}`).toBe(200);
+                expect(response.status, `Expected > 400 OK response for image ${links[i]}`).toBeLessThan(400);
                 console.log(`${links[i]}: ✅`)
         })
         .catch((error) =>  {
@@ -81,6 +150,9 @@ test('Crawl for bad URIs', async () => {
   await crawl(baseUrl + startPath);
   console.log('Crawl checked', crawled.length);
   console.log(`Failed URLS: ${failed_urls}`)
+
+  generate_index_html(failed_urls);
+
   // Close the browser
   await browser.close();
 });
@@ -102,9 +174,9 @@ async function crawlImages() {
 
   for (let i = 0; i < images.length; i++) {
     if (!all_images.includes(images[i])){
-        fetch(images[i])
+        fetch_url(images[i])
         .then((response) => {
-            expect(response.status, `Expected a 200 OK response for image ${images[i]}`).toBe(200);
+            expect(response.status, `Expected less than 400 OK response for image ${images[i]}`).toBeLessThan(400);
             console.log(`${images[i]}: ✅`)
 
         })
@@ -125,6 +197,7 @@ async function crawlImages() {
 }
 
 function addUri(collection, uri) {
+  //console.log(uri)
 
 
   if (uri.substring(0, 1) == '/') {
@@ -138,7 +211,7 @@ function addUri(collection, uri) {
     if (uri.indexOf("https://") == 0 && !external_links.includes(uri)) {
         external_links.push(uri)
 
-    fetch(uri).then((response) => {
+    fetch_url(uri).then((response) => {
 
             expect(response.status, `Expected a successful HTTP response for external URL ${uri}`).toBeLessThan(404);
             console.log(`External URL ${uri}: ✅`);

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -13,8 +13,10 @@
   "license": "ISC",
   "dependencies": {
     "@playwright/test": "^1.40.1",
+    "fs": "^0.0.1-security",
     "rehype-parse": "^9.0.0",
     "rehype-stringify": "^10.0.0",
+    "undici": "^6.0.1",
     "unified": "^11.0.4",
     "unist-util-visit": "^5.0.0"
   },