Skip to content

Commit

Permalink
feat: Add Kotlin to Dependency Graph Integrator (#1227)
Browse files Browse the repository at this point in the history
* feat: refactor file generator to take language as parameter

* feat: add types

* feat: refactor sns event creation to work with more than one language

* feat: update dep graph integrator to take language paramater

* refactor: remove explicit reference to Scala

* fix: add language param to createYaml function

* fix: add language to event for running locally

* refactor: tidy up types

* refactor: add further information section to PR body

* fix: type error

* refactor: split out yaml steps for each language

* fix: update tests

* refactor: feedback

* feat: stop creating dependency graph integrator PRs via repocop (#1240)
  • Loading branch information
tjsilver authored Aug 14, 2024
1 parent 4fa5c34 commit b230aeb
Show file tree
Hide file tree
Showing 9 changed files with 234 additions and 83 deletions.
3 changes: 3 additions & 0 deletions packages/common/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@ export type GitHubAppConfig = {
installationId: string;
};

export type DepGraphLanguage = 'Scala' | 'Kotlin';

export interface DependencyGraphIntegratorEvent {
name: string;
language: DepGraphLanguage;
}

export interface SnykIntegratorEvent {
Expand Down
45 changes: 42 additions & 3 deletions packages/dependency-graph-integrator/src/file-generator.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { createYaml } from './file-generator';

describe('createYaml', () => {
describe('createYaml for sbt', () => {
it('should generate the following yaml file', () => {
const yaml = createYaml('branch');
const yaml = createYaml('branch', 'Scala', 'repo1');
const result =
String.raw`name: Update Dependency Graph for SBT
String.raw`name: Update Dependency Graph for sbt
on:
push:
branches:
Expand Down Expand Up @@ -32,3 +32,42 @@ jobs:
expect(yaml).toEqual(result);
});
});

describe('createYaml for Kotlin', () => {
it('should generate the following yaml file', () => {
const yaml = createYaml('branch', 'Kotlin', 'repo2');
const result =
String.raw`name: Update Dependency Graph for Gradle
on:
push:
branches:
- main
- branch
workflow_dispatch:
jobs:
dependency-graph:
runs-on: ubuntu-latest
steps:
- name: Checkout branch
id: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Set up Java
id: setup
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
distribution: temurin
java-version: 17
- name: Submit dependencies
id: submit
uses: gradle/actions/dependency-submission@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0
- name: Log snapshot for user validation
id: validate
run: cat ` + // Need to split this line to avoid errors due to new line produced in yaml
'/home/runner/work/repo2/repo2/dependency-graph-reports/update_dependency_graph_for_kotlin-dependency-graph.json\n | jq' +
String.raw`
permissions:
contents: write
`;
expect(yaml).toEqual(result);
});
});
135 changes: 100 additions & 35 deletions packages/dependency-graph-integrator/src/file-generator.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,70 @@
import { markdownChecklist } from 'common/src/string';
import type { DepGraphLanguage } from 'common/types';
import { h2, p, tsMarkdown } from 'ts-markdown';
import { stringify } from 'yaml';

export function createYaml(prBranch: string): string {
export const depGraphPackageManager: Record<DepGraphLanguage, string> = {
Scala: 'sbt',
Kotlin: 'Gradle',
};

function createLanguageSpecificWorkflowSteps(
repo: string,
): Record<DepGraphLanguage, ConcatArray<object>> {
return {
Scala: [
{
name: 'Checkout branch',
id: 'checkout',
uses: 'actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7',
},
{
name: 'Submit dependencies',
id: 'submit',
uses: 'scalacenter/sbt-dependency-submission@7ebd561e5280336d3d5b445a59013810ff79325e # v3.0.1',
},
{
name: 'Log snapshot for user validation',
id: 'validate',
run: 'cat ${{ steps.submit.outputs.snapshot-json-path }} | jq',
},
],
Kotlin: [
{
name: 'Checkout branch',
id: 'checkout',
uses: 'actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7',
},
{
name: 'Set up Java',
id: 'setup',
uses: 'actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1',
with: {
distribution: 'temurin',
'java-version': '17',
},
},
{
name: 'Submit dependencies',
id: 'submit',
uses: 'gradle/actions/dependency-submission@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0',
},
{
name: 'Log snapshot for user validation',
id: 'validate',
run: `cat /home/runner/work/${repo}/${repo}/dependency-graph-reports/update_dependency_graph_for_kotlin-dependency-graph.json | jq`,
},
],
};
}

export function createYaml(
prBranch: string,
language: DepGraphLanguage,
repo: string,
): string {
const dependencyGraphWorkflowJson = {
name: 'Update Dependency Graph for SBT',
name: `Update Dependency Graph for ${depGraphPackageManager[language]}`,
on: {
push: {
branches: ['main', prBranch],
Expand All @@ -14,62 +74,63 @@ export function createYaml(prBranch: string): string {
jobs: {
'dependency-graph': {
'runs-on': 'ubuntu-latest',
steps: [
{
name: 'Checkout branch',
id: 'checkout',
uses: 'actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7',
},
{
name: 'Submit dependencies',
id: 'submit',
uses: 'scalacenter/sbt-dependency-submission@7ebd561e5280336d3d5b445a59013810ff79325e # v3.0.1',
},
{
name: 'Log snapshot for user validation',
id: 'validate',
run: 'cat ${{ steps.submit.outputs.snapshot-json-path }} | jq',
},
],
steps: createLanguageSpecificWorkflowSteps(repo)[language],
permissions: { contents: 'write' },
},
},
};

return stringify(dependencyGraphWorkflowJson, { lineWidth: 120 })
.replace('{}', '')
.replaceAll('{}', '')
.replaceAll('"', '');
}

function createPRChecklist(branchName: string): string[] {
const step1 =
'Ensure that the [version of sbt in the project is v1.5 or above](https://github.com/scalacenter/sbt-dependency-submission?tab=readme-ov-file#support) in order for the dependency submission action to run.';
const stepsForLanguages: Record<DepGraphLanguage, string> = {
Scala:
'Ensure that the [version of sbt in the project is v1.5 or above](https://github.com/scalacenter/sbt-dependency-submission?tab=readme-ov-file#support) in order for the dependency submission action to run.',

const step2 =
'A run of this action should have been triggered when the branch was ' +
'created. Sense check the output of "Log snapshot for user validation", ' +
'and make sure that your dependencies look okay.';
Kotlin:
'Ensure that the [version of Gradle is v5.2 or above](https://github.com/gradle/actions/blob/main/docs/dependency-submission.md#gradle-version-compatibility)',
};

const step3 =
`When you are happy the action works, remove the branch name \`${branchName}\` ` +
'trigger from the the yaml file (aka delete line 6), approve, and merge. ';
const languageSpecificInfo: Record<DepGraphLanguage, string> = {
Scala:
'See [the sbt workflow documentation](https://github.com/scalacenter/sbt-dependency-submission?tab=readme-ov-file) for further information and configuration options.',
Kotlin:
'See [the Gradle workflow documentation](https://github.com/gradle/actions/blob/main/docs/dependency-submission.md) for further information and configuration options, and the [FAQ for troubleshooting](https://github.com/gradle/actions/blob/main/docs/dependency-submission-faq.md).',
};

return [step1, step2, step3];
function createPRChecklist(
branchName: string,
stepsForLanguage: string,
): string[] {
const finalSteps = [
'A run of this action should have been triggered when the branch was ' +
'created. Sense check the output of "Log snapshot for user validation", ' +
'and make sure that your dependencies look okay.',
`When you are happy the action works, remove the branch name \`${branchName}\` ` +
'trigger from the the yaml file (aka delete line 6), approve, and merge. ',
];
return [stepsForLanguage, ...finalSteps];
}

export function generatePrBody(branchName: string, repoName: string): string {
export function generatePrBody(
branchName: string,
repoName: string,
language: DepGraphLanguage,
): string {
const body = [
h2('What does this change?'),
p(
'This PR sends your sbt dependencies to GitHub for vulnerability monitoring via Dependabot. ' +
`This PR sends your ${depGraphPackageManager[language]} dependencies to GitHub for vulnerability monitoring via Dependabot. ` +
`The submitted dependencies will appear in the [Dependency Graph](https://github.com/guardian/${repoName}/network/dependencies) ` +
'on merge to main (it might take a few minutes to update).',
),
h2('Why?'),
p(
'If a repository is in production, we need to track its third party dependencies for vulnerabilities. ' +
'Historically, we have done this using Snyk, but we are now moving to GitHub’s native Dependabot. ' +
'Scala is not a language that Dependabot supports out of the box, this workflow is required to make it happen. ' +
`${language} is not a language that Dependabot supports out of the box, this workflow is required to make it happen. ` +
'As a result, we have raised this PR on your behalf to add it to the Dependency Graph.',
),
h2('How has it been verified?'),
Expand All @@ -78,8 +139,12 @@ export function generatePrBody(branchName: string, repoName: string): string {
'However, we have included some instructions below to help you verify that it works for you. ' +
'Please do not hesitate to contact DevX Security if you have any questions or concerns.',
),
h2(`Further information for ${depGraphPackageManager[language]}`),
p(languageSpecificInfo[`${language}`]),
h2('What do I need to do?'),
markdownChecklist(createPRChecklist(branchName)),
markdownChecklist(
createPRChecklist(branchName, stepsForLanguages[`${language}`]),
),
];
return tsMarkdown(body);
}
34 changes: 21 additions & 13 deletions packages/dependency-graph-integrator/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,36 @@ import { generateBranchName } from 'common/src/pull-requests';
import type { DependencyGraphIntegratorEvent } from 'common/src/types';
import type { Config } from './config';
import { getConfig } from './config';
import { createYaml, generatePrBody } from './file-generator';
import {
createYaml,
depGraphPackageManager,
generatePrBody,
} from './file-generator';
import {
createPrAndAddToProject,
enableDependabotAlerts,
} from './repo-functions';
import type { StatusCode } from './types';

export async function main(event: DependencyGraphIntegratorEvent) {
console.log(`Generating Dependabot PR for ${event.name}`);
const language = event.language;
const name = event.name;
console.log(
`Generating Dependabot PR for ${name} repo with ${language} language`,
);
const config: Config = getConfig();
const branch = generateBranchName('sbt-dependency-graph');

const boardNumber = 110;
const author = 'gu-dependency-graph-integrator'; // TODO: create new 'gu-dependency-graph-integrator' app
const title =
'Submit sbt dependencies to GitHub for vulnerability monitoring';
const fileName = '.github/workflows/sbt-dependency-graph.yaml';
const commitMessage = 'Add sbt-dependency-graph.yaml';
const yamlContents = createYaml(branch);
const repo = event.name;
const prContents = generatePrBody(branch, repo);
const stage = config.stage;
const branch = generateBranchName(
`${depGraphPackageManager[language]}-dependency-graph`,
);
const boardNumber = 110;
const author = 'gu-dependency-graph-integrator';
const title = `Submit ${depGraphPackageManager[language]} dependencies to GitHub for vulnerability monitoring`;
const fileName = `.github/workflows/${depGraphPackageManager[language].toLowerCase()}-dependency-graph.yaml`;
const commitMessage = `Add ${depGraphPackageManager[language].toLowerCase()}-dependency-graph.yaml`;
const yamlContents = createYaml(branch, language, name);
const repo = name;
const prContents = generatePrBody(branch, repo, language);

if (stage === 'PROD') {
const octokit = await stageAwareOctokit(stage);
Expand Down
1 change: 1 addition & 0 deletions packages/dependency-graph-integrator/src/run-locally.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ config({ path: `${homedir()}/.gu/service_catalogue/.env.local` });
if (require.main === module) {
void main({
name: 'service-catalogue',
language: 'Scala',
});
}
6 changes: 6 additions & 0 deletions packages/repocop/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ export interface Config extends PrismaConfig {
*/
snykIntegratorTopic: string;

/**
* Flag to enable creation of Depedency Graph integration PRs
*/
depGraphIntegrationPREnabled: boolean;

/**
* The ARN of the Dependency Graph Integrator input topic.
*/
Expand Down Expand Up @@ -97,6 +102,7 @@ export async function getConfig(): Promise<Config> {
snykIntegrationPREnabled:
process.env.SNYK_INTEGRATION_PR_ENABLED === 'true',
snykIntegratorTopic: getEnvOrThrow('SNYK_INTEGRATOR_INPUT_TOPIC_ARN'),
depGraphIntegrationPREnabled: false,
dependencyGraphIntegratorTopic: getEnvOrThrow(
'DEPENDENCY_GRAPH_INPUT_TOPIC_ARN',
),
Expand Down
18 changes: 12 additions & 6 deletions packages/repocop/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,12 +196,18 @@ export async function main() {
);
}

await sendOneRepoToDepGraphIntegrator(
config,
repoLanguages,
productionRepos,
productionWorkflowUsages,
);
if (config.depGraphIntegrationPREnabled) {
await sendOneRepoToDepGraphIntegrator(
config,
repoLanguages,
productionRepos,
productionWorkflowUsages,
);
} else {
console.log(
'Messaging to Dependency Graph Integrator is not enabled, use console to send SNS message and raise a PR if required.',
);
}

console.log('Done');
}
Loading

0 comments on commit b230aeb

Please sign in to comment.