Skip to content

Commit

Permalink
pretalx sync
Browse files Browse the repository at this point in the history
  • Loading branch information
wslyvh committed Oct 15, 2024
1 parent cbaecf5 commit f8274c9
Show file tree
Hide file tree
Showing 8 changed files with 109 additions and 28 deletions.
23 changes: 23 additions & 0 deletions .github/workflows/sync-pretalx.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Pretalx Sync

on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- uses: actions/setup-node@v1
with:
node-version: "20"
- run: yarn install
- run: yarn sync:pretalx
env:
PRETALX_API_KEY: ${{ secrets.PRETALX_API_KEY }}
- uses: EndBug/add-and-commit@v9
with:
default_author: github_actions
message: "[action] Pretalx Sync"
2 changes: 1 addition & 1 deletion devcon-api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"lint": "eslint --fix",
"prettier": "prettier './src' --write",
"slides": "ts-node ./src/scripts/slides.ts",
"seed:pretalx": "ts-node ./src/scripts/seed-pretalx.ts",
"sync:pretalx": "ts-node ./src/scripts/sync-pretalx.ts",
"db:generate": "prisma db push --force-reset --accept-data-loss && prisma db seed && yarn db:generate:account",
"db:generate:account": "prisma db push --schema src/db/account.prisma && yarn copy:account",
"copy:account": "mkdir -p dist/db && cp src/db/clients dist/db/clients -r",
Expand Down
3 changes: 2 additions & 1 deletion devcon-api/src/clients/pretalx.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export async function GetSpeaker(id: string, params: Partial<RequestParams> = {}

async function exhaustResource(slug: string, limit = PRETALX_CONFIG.DEFAULT_LIMIT, offset = 0, results = [] as any): Promise<any> {
return get(`${slug}${slug.includes('?') ? '&' : '?'}limit=${limit}&offset=${offset}`).then((data: any) => {
return data.results
results.push(data.results)
if (data.next) {
console.log('GET', slug, 'TOTAL COUNT', data.count)
Expand Down Expand Up @@ -114,7 +115,7 @@ function mapSession(i: any, params: Partial<RequestParams>) {
keywords: keywords,
tags: [...i.tags, ...predefinedTags] ?? [],
language: 'en',
speakers: params.inclContacts ? i.speakers.map((i: any) => mapSpeaker(i, params)) : i.speakers.map((i: any) => i.code),
speakers: params.inclContacts ? i.speakers.map((i: any) => mapSpeaker(i, params)) : i.speakers.map((i: any) => defaultSlugify(i.name)),
eventId: `devcon-${PRETALX_CONFIG.PRETALX_EVENT_ID}`,
}

Expand Down
8 changes: 2 additions & 6 deletions devcon-api/src/db/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { GetData, GetSpeakerData } from '@/clients/filesystem'
import { PrismaClient } from '@prisma/client'
import words from 'natural/lib/natural/util'
import dayjs from 'dayjs'
import { seedPretalx } from './pretalx'

const client = new PrismaClient()
const stopwords = [
Expand Down Expand Up @@ -295,7 +294,8 @@ async function main() {

let data: any = {
...item,
tags: item.tags.join(','),
tags: item.tags?.join(',') || '',
keywords: item.keywords?.join(',') || '',
slot_start: item.slot_start ? dayjs(item.slot_start).toISOString() : null,
slot_end: item.slot_end ? dayjs(item.slot_end).toISOString() : null,
event: {
Expand Down Expand Up @@ -379,10 +379,6 @@ async function main() {
await client.relatedSession.createMany({
data: relatedSessions,
})

// Seed Pretalx Data
console.log('- Seed Pretalx Data')
await seedPretalx()
}

function computeCosineSimilarity(vector1: number[], vector2: number[]): number {
Expand Down
14 changes: 0 additions & 14 deletions devcon-api/src/scripts/seed-pretalx.ts

This file was deleted.

80 changes: 80 additions & 0 deletions devcon-api/src/scripts/sync-pretalx.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { GetData } from '@/clients/filesystem'
import { GetRooms, GetSpeakers, GetSubmissions } from '@/clients/pretalx'
import fs from 'fs'

async function main() {
console.log('Syncing Pretalx...')
await syncRooms()
await syncSessions()
}

async function syncRooms() {
if (!fs.existsSync('./data/rooms/devcon-7')) {
fs.mkdirSync('./data/rooms/devcon-7')
}

const rooms = await GetRooms()
const roomsFs = GetData('rooms/devcon-7')
console.log('Rooms Pretalx', rooms.length, 'Rooms fs', roomsFs.length)

console.log('Sync Rooms')
for (const room of roomsFs) {
if (!rooms.some((r: any) => r.id === room.id)) {
console.log('- delete room', room.id)
fs.unlinkSync(`./data/rooms/devcon-7/${room.id}.json`)
}
}

for (const room of rooms) {
fs.writeFileSync(`./data/rooms/devcon-7/${room.id}.json`, JSON.stringify(room, null, 2))
}

console.log('Synced Pretalx Rooms')
console.log('')
}

async function syncSessions() {
const speakers = await GetSpeakers()
const acceptedSpeakers: any[] = []

if (!fs.existsSync(`./data/sessions/devcon-7`)) {
fs.mkdirSync(`./data/sessions/devcon-7`)
}
const sessions = await GetSubmissions()
const sessionsFs = GetData('sessions/devcon-7')
console.log('Sessions Pretalx', sessions.length, 'Sessions fs', sessionsFs.length)

console.log('Sync Sessions')
for (const session of sessionsFs) {
if (!sessions.some((s: any) => s.id === session.id)) {
console.log('- delete session', session.id)
fs.unlinkSync(`./data/sessions/devcon-7/${session.id}.json`)
}
}

for (const session of sessions) {
if (session.speakers.length > 0) {
acceptedSpeakers.push(...speakers.filter((s: any) => session.speakers.includes(s.id)))
}

fs.writeFileSync(`./data/sessions/devcon-7/${session.id}.json`, JSON.stringify(session, null, 2))
}

console.log('Speakers Pretalx', speakers.length, 'Accepted Speakers', acceptedSpeakers.length)
console.log('Sync Speakers')
for (const speaker of acceptedSpeakers) {
fs.writeFileSync(`./data/speakers/${speaker.id}.json`, JSON.stringify(speaker, null, 2))
}

console.log('Synced Pretalx Schedule')
console.log('')
}

main()
.then(async () => {
console.log('All done!')
})
.catch(async (e) => {
console.error(e)
process.exit(1)
})
5 changes: 0 additions & 5 deletions devcon-api/src/swagger/definition.json
Original file line number Diff line number Diff line change
Expand Up @@ -937,11 +937,6 @@
"in": "header",
"type": "string"
},
{
"name": "recommendations",
"in": "query",
"type": "string"
},
{
"name": "body",
"in": "body",
Expand Down
2 changes: 1 addition & 1 deletion devcon-api/src/utils/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export const PRETALX_CONFIG = {
PRETALX_QUESTIONS_TAGS: 76,
PRETALX_QUESTIONS_KEYWORDS: 73,

DEFAULT_LIMIT: 100,
DEFAULT_LIMIT: 20,
}

export const SESSION_CONFIG = {
Expand Down

0 comments on commit f8274c9

Please sign in to comment.