diff --git a/.github/workflows/sync-pretalx.yml b/.github/workflows/sync-pretalx.yml new file mode 100644 index 000000000..e28ef72c2 --- /dev/null +++ b/.github/workflows/sync-pretalx.yml @@ -0,0 +1,23 @@ +name: Pretalx Sync + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-node@v1 + with: + node-version: "20" + - run: yarn install + - run: yarn sync:pretalx + env: + PRETALX_API_KEY: ${{ secrets.PRETALX_API_KEY }} + - uses: EndBug/add-and-commit@v9 + with: + default_author: github_actions + message: "[action] Pretalx Sync" diff --git a/devcon-api/package.json b/devcon-api/package.json index 891d9a72c..39f0b4949 100644 --- a/devcon-api/package.json +++ b/devcon-api/package.json @@ -19,7 +19,7 @@ "lint": "eslint --fix", "prettier": "prettier './src' --write", "slides": "ts-node ./src/scripts/slides.ts", - "seed:pretalx": "ts-node ./src/scripts/seed-pretalx.ts", + "sync:pretalx": "ts-node ./src/scripts/sync-pretalx.ts", "db:generate": "prisma db push --force-reset --accept-data-loss && prisma db seed && yarn db:generate:account", "db:generate:account": "prisma db push --schema src/db/account.prisma && yarn copy:account", "copy:account": "mkdir -p dist/db && cp src/db/clients dist/db/clients -r", diff --git a/devcon-api/src/clients/pretalx.ts b/devcon-api/src/clients/pretalx.ts index a7a1e82b3..18b3537b3 100644 --- a/devcon-api/src/clients/pretalx.ts +++ b/devcon-api/src/clients/pretalx.ts @@ -66,6 +66,7 @@ export async function GetSpeaker(id: string, params: Partial = {} async function exhaustResource(slug: string, limit = PRETALX_CONFIG.DEFAULT_LIMIT, offset = 0, results = [] as any): Promise { return get(`${slug}${slug.includes('?') ? '&' : '?'}limit=${limit}&offset=${offset}`).then((data: any) => { + return data.results results.push(data.results) if (data.next) { console.log('GET', slug, 'TOTAL COUNT', data.count) @@ -114,7 +115,7 @@ function mapSession(i: any, params: Partial) { keywords: keywords, tags: [...i.tags, ...predefinedTags] ?? [], language: 'en', - speakers: params.inclContacts ? i.speakers.map((i: any) => mapSpeaker(i, params)) : i.speakers.map((i: any) => i.code), + speakers: params.inclContacts ? i.speakers.map((i: any) => mapSpeaker(i, params)) : i.speakers.map((i: any) => defaultSlugify(i.name)), eventId: `devcon-${PRETALX_CONFIG.PRETALX_EVENT_ID}`, } diff --git a/devcon-api/src/db/seed.ts b/devcon-api/src/db/seed.ts index f889ae886..23c03aa1c 100644 --- a/devcon-api/src/db/seed.ts +++ b/devcon-api/src/db/seed.ts @@ -3,7 +3,6 @@ import { GetData, GetSpeakerData } from '@/clients/filesystem' import { PrismaClient } from '@prisma/client' import words from 'natural/lib/natural/util' import dayjs from 'dayjs' -import { seedPretalx } from './pretalx' const client = new PrismaClient() const stopwords = [ @@ -295,7 +294,8 @@ async function main() { let data: any = { ...item, - tags: item.tags.join(','), + tags: item.tags?.join(',') || '', + keywords: item.keywords?.join(',') || '', slot_start: item.slot_start ? dayjs(item.slot_start).toISOString() : null, slot_end: item.slot_end ? dayjs(item.slot_end).toISOString() : null, event: { @@ -379,10 +379,6 @@ async function main() { await client.relatedSession.createMany({ data: relatedSessions, }) - - // Seed Pretalx Data - console.log('- Seed Pretalx Data') - await seedPretalx() } function computeCosineSimilarity(vector1: number[], vector2: number[]): number { diff --git a/devcon-api/src/scripts/seed-pretalx.ts b/devcon-api/src/scripts/seed-pretalx.ts deleted file mode 100644 index 1db25e343..000000000 --- a/devcon-api/src/scripts/seed-pretalx.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { seedPretalx } from '@/db/pretalx' - -async function main() { - await seedPretalx() -} - -main() - .then(async () => { - console.log('All done!') - }) - .catch(async (e) => { - console.error(e) - process.exit(1) - }) diff --git a/devcon-api/src/scripts/sync-pretalx.ts b/devcon-api/src/scripts/sync-pretalx.ts new file mode 100644 index 000000000..8c42e0f94 --- /dev/null +++ b/devcon-api/src/scripts/sync-pretalx.ts @@ -0,0 +1,80 @@ +import { GetData } from '@/clients/filesystem' +import { GetRooms, GetSpeakers, GetSubmissions } from '@/clients/pretalx' +import fs from 'fs' + +async function main() { + console.log('Syncing Pretalx...') + await syncRooms() + await syncSessions() +} + +async function syncRooms() { + if (!fs.existsSync('./data/rooms/devcon-7')) { + fs.mkdirSync('./data/rooms/devcon-7') + } + + const rooms = await GetRooms() + const roomsFs = GetData('rooms/devcon-7') + console.log('Rooms Pretalx', rooms.length, 'Rooms fs', roomsFs.length) + + console.log('Sync Rooms') + for (const room of roomsFs) { + if (!rooms.some((r: any) => r.id === room.id)) { + console.log('- delete room', room.id) + fs.unlinkSync(`./data/rooms/devcon-7/${room.id}.json`) + } + } + + for (const room of rooms) { + fs.writeFileSync(`./data/rooms/devcon-7/${room.id}.json`, JSON.stringify(room, null, 2)) + } + + console.log('Synced Pretalx Rooms') + console.log('') +} + +async function syncSessions() { + const speakers = await GetSpeakers() + const acceptedSpeakers: any[] = [] + + if (!fs.existsSync(`./data/sessions/devcon-7`)) { + fs.mkdirSync(`./data/sessions/devcon-7`) + } + const sessions = await GetSubmissions() + const sessionsFs = GetData('sessions/devcon-7') + console.log('Sessions Pretalx', sessions.length, 'Sessions fs', sessionsFs.length) + + console.log('Sync Sessions') + for (const session of sessionsFs) { + if (!sessions.some((s: any) => s.id === session.id)) { + console.log('- delete session', session.id) + fs.unlinkSync(`./data/sessions/devcon-7/${session.id}.json`) + } + } + + for (const session of sessions) { + if (session.speakers.length > 0) { + acceptedSpeakers.push(...speakers.filter((s: any) => session.speakers.includes(s.id))) + } + + fs.writeFileSync(`./data/sessions/devcon-7/${session.id}.json`, JSON.stringify(session, null, 2)) + } + + console.log('Speakers Pretalx', speakers.length, 'Accepted Speakers', acceptedSpeakers.length) + console.log('Sync Speakers') + for (const speaker of acceptedSpeakers) { + fs.writeFileSync(`./data/speakers/${speaker.id}.json`, JSON.stringify(speaker, null, 2)) + } + + console.log('Synced Pretalx Schedule') + console.log('') +} + +main() + .then(async () => { + console.log('All done!') + }) + .catch(async (e) => { + console.error(e) + process.exit(1) + }) diff --git a/devcon-api/src/swagger/definition.json b/devcon-api/src/swagger/definition.json index 47099d938..ee2b2072c 100644 --- a/devcon-api/src/swagger/definition.json +++ b/devcon-api/src/swagger/definition.json @@ -937,11 +937,6 @@ "in": "header", "type": "string" }, - { - "name": "recommendations", - "in": "query", - "type": "string" - }, { "name": "body", "in": "body", diff --git a/devcon-api/src/utils/config.ts b/devcon-api/src/utils/config.ts index 533854f48..c85ded2c6 100644 --- a/devcon-api/src/utils/config.ts +++ b/devcon-api/src/utils/config.ts @@ -73,7 +73,7 @@ export const PRETALX_CONFIG = { PRETALX_QUESTIONS_TAGS: 76, PRETALX_QUESTIONS_KEYWORDS: 73, - DEFAULT_LIMIT: 100, + DEFAULT_LIMIT: 20, } export const SESSION_CONFIG = {