Initial commit containing v0.1.0 of Soko

This is a rewrite of packages.g.o based on postgres and git
written in Go.

Signed-off-by: Max Magorsch <[email protected]>

.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+assets/
+.idea
+
+.bin/soko

.gitlab-ci.yml

@@ -0,0 +1,45 @@
+stages:
+  - build
+  - test
+  - dast
+
+info:
+  stage: .pre
+  script:
+    - docker info
+
+build:
+  stage: build
+  variables:
+    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:$CI_COMMIT_SHA
+    UPDATER_IMAGE_TAG: $CI_REGISTRY_IMAGE-updater/$CI_COMMIT_BRANCH:$CI_COMMIT_SHA
+    LATEST_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:latest
+    LATEST_UPDATER_IMAGE_TAG: $CI_REGISTRY_IMAGE-updater/$CI_COMMIT_BRANCH:latest
+  script:
+    - echo $IMAGE_TAG
+    - echo $UPDATER_IMAGE_TAG
+    - echo $LATEST_IMAGE_TAG
+    - echo $LATEST_UPDATER_IMAGE_TAG
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
+    - docker pull gentoo/portage:latest
+    - docker pull gentoo/stage3-amd64
+    - docker pull $LATEST_IMAGE_TAG || true
+    - docker pull $LATEST_UPDATER_IMAGE_TAG || true
+    - docker build --cache-from $LATEST_IMAGE_TAG -t $IMAGE_TAG -t $LATEST_IMAGE_TAG .
+    - docker build --cache-from $LATEST_UPDATER_IMAGE_TAG -t $UPDATER_IMAGE_TAG -t $LATEST_UPDATER_IMAGE_TAG -f Dockerfile.updater .
+    - docker push $LATEST_IMAGE_TAG
+    - docker push $IMAGE_TAG
+    - docker push $UPDATER_IMAGE_TAG
+    - docker push $LATEST_UPDATER_IMAGE_TAG
+
+include:
+  - template: Dependency-Scanning.gitlab-ci.yml
+  - template: Container-Scanning.gitlab-ci.yml  
+  - template: SAST.gitlab-ci.yml
+  - template: DAST.gitlab-ci.yml
+
+variables:
+  DS_DISABLE_DIND: "true"
+  SAST_DISABLE_DIND: "true"
+  DAST_WEBSITE: https://packagestest.gentoo.org
+#  DAST_FULL_SCAN_ENABLED: "true"

Dockerfile

@@ -0,0 +1,19 @@
+FROM golang:1.14.0 AS builder
+WORKDIR /go/src/soko
+COPY . /go/src/soko
+RUN go get github.com/go-pg/pg/v9
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o bin .
+
+FROM node:13 AS assetsbuilder
+WORKDIR /go/src/soko
+COPY . /go/src/soko
+RUN npm install && cd node_modules/@gentoo/tyrian && npm install && npm run dist && cd /go/src/soko
+RUN npx webpack
+
+FROM scratch
+WORKDIR /go/src/soko
+COPY --from=assetsbuilder /go/src/soko/assets /go/src/soko/assets
+COPY --from=builder /go/src/soko/bin /go/src/soko/bin
+COPY --from=builder /go/src/soko/pkg /go/src/soko/pkg
+COPY --from=builder /go/src/soko/web /go/src/soko/web
+ENTRYPOINT ["/go/src/soko/bin/soko", "serve"]

Dockerfile.updater

@@ -0,0 +1,27 @@
+FROM golang:1.14.0 AS builder
+WORKDIR /go/src/soko
+COPY . /go/src/soko
+RUN go get github.com/go-pg/pg/v9
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o bin .
+
+
+FROM gentoo/stage3-amd64 as gentoobuilder
+# Need a portage tree to build, use last nights.
+COPY --from=gentoo/portage:latest /var/db/repos/gentoo /var/db/repos/gentoo
+# Sandbox doesn't work well in docker.
+ENV FEATURES="-userpriv -usersandbox -sandbox"
+ENV USE="-bindist"
+RUN emerge -C openssh
+RUN emerge --quiet-build dev-vcs/git
+RUN echo 'FEATURES="-userpriv -usersandbox -sandbox"' >> /etc/portage/make.conf
+RUN rm -rf /var/db/repos/gentoo
+
+
+FROM scratch
+COPY --from=gentoobuilder / /
+COPY --from=builder /go/src/soko/assets /go/src/soko/assets
+COPY --from=builder /go/src/soko/bin /go/src/soko/bin
+COPY --from=builder /go/src/soko/pkg /go/src/soko/pkg
+COPY --from=builder /go/src/soko/web /go/src/soko/web
+WORKDIR /go/src/soko
+ENTRYPOINT ["/go/src/soko/bin/update.sh"]

bin/update.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+update_repository(){
+  # This is the copy of the tree used to run gpackages against.
+  if [[ ! -d /mnt/packages-tree/gentoo/ ]]; then
+      cd /mnt/packages-tree || exit 1
+      git clone https://anongit.gentoo.org/git/repo/gentoo.git
+  else
+      cd /mnt/packages-tree/gentoo/ || exit 1
+      git pull --rebase &>/dev/null
+  fi
+}
+
+update_md5cache(){
+  mkdir -p /var/cache/pgo-egencache
+  cd /mnt/packages-tree/gentoo/ || exit 1
+
+  #echo 'FEATURES="-userpriv -usersandbox -sandbox"' >> /etc/portage/make.conf
+
+  egencache -j 6 --cache-dir /var/cache/pgo-egencache --repo gentoo --repositories-configuration '[gentoo]
+  location = /mnt/packages-tree/gentoo' --update
+
+  egencache -j 6 --cache-dir /var/cache/pgo-egencache --repo gentoo --repositories-configuration '[gentoo]
+  location = /mnt/packages-tree/gentoo' --update-use-local-desc
+}
+
+update_database(){
+  cd /mnt/packages-tree/gentoo/ || exit 1
+  /go/src/soko/bin/soko update
+}
+
+
+update_repository
+update_md5cache
+update_database

docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3.2'
+
+services:
+  http-serving:
+    image: ${SOKO_IMAGE:-gentoo/soko:latest}
+    ports:
+      - 127.0.0.1:5000:5000
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"
+    restart: always
+    depends_on:
+      - db
+  updater:
+    image: ${SOKO_UPDATER_IMAGE:-gentoo/soko-updater:latest}
+    volumes:
+      - type: "bind"
+        source: "/mnt/packages-tree"
+        target: "/mnt/packages-tree"
+      - type: "bind"
+        source: "/var/cache/pgo-egencache"
+        target: "/var/cache/pgo-egencache"
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"
+    restart: always
+    depends_on:
+      - db
+  db:
+    image: postgres:12
+    restart: always
+    environment:
+      POSTGRES_USER: ${SOKO_POSTGRES_USER:-root}
+      POSTGRES_PASSWORD: ${SOKO_POSTGRES_PASSWORD:-root}
+      POSTGRES_DB: ${SOKO_POSTGRES_DB:-soko}
+    volumes:
+      - ${POSTGRES_DATA_PATH:-/var/lib/postgresql/data}:/var/lib/postgresql/data
+  watchtower:
+    image: containrrr/watchtower:0.3.10
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /root/.docker/config.json:/config.json
+    command: --label-enable

go.mod

@@ -0,0 +1,8 @@
+module soko
+
+go 1.13
+
+require (
+	github.com/go-pg/pg v8.0.6+incompatible
+	github.com/go-pg/pg/v9 v9.1.3
+)

go.sum

@@ -0,0 +1,75 @@
+github.com/codemodus/kace v0.5.1 h1:4OCsBlE2c/rSJo375ggfnucv9eRzge/U5LrrOZd47HA=
+github.com/codemodus/kace v0.5.1/go.mod h1:coddaHoX1ku1YFSe4Ip0mL9kQjJvKkzb9CfIdG1YR04=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-pg/pg v8.0.6+incompatible h1:Hi7yUJ2zwmHFq1Mar5XqhCe3NJ7j9r+BaiNmd+vqf+A=
+github.com/go-pg/pg v8.0.6+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
+github.com/go-pg/pg/v9 v9.0.0-beta.14/go.mod h1:T2Sr6bpTCOr2lUqOUMiXLMJqZHSUBKk1LdgSqjwhZfA=
+github.com/go-pg/pg/v9 v9.0.3/go.mod h1:Tm/Q3Vt6gdQOH6TTN1H/xLlIXc+Qrka7TZ6uREtu/eA=
+github.com/go-pg/pg/v9 v9.1.3 h1:gmE7k5ib45+NcRJBGUDPD/keJGCMqhH7TPqbd9xbdz4=
+github.com/go-pg/pg/v9 v9.1.3/go.mod h1:QM13HBLkdml4zcKOfUfGLymM6hb72aKTJLrmaH8rsFg=
+github.com/go-pg/urlstruct v0.1.0/go.mod h1:2Nag+BIny6G/KYCkdt++ZnqU/VinzimGapKfs4kwlN0=
+github.com/go-pg/urlstruct v0.2.6/go.mod h1:dxENwVISWSOX+k87hDt0ueEJadD+gZWv3tHzwfmZPu8=
+github.com/go-pg/urlstruct v0.3.0 h1:ORiTb205uT7v7lvq4VUOQ4ddJ5TjFRlVRWqWsMBgek4=
+github.com/go-pg/urlstruct v0.3.0/go.mod h1:/XKyiUOUUS3onjF+LJxbfmSywYAdl6qMfVbX33Q8rgg=
+github.com/go-pg/zerochecker v0.1.1 h1:av77Qe7Gs+1oYGGh51k0sbZ0bUaxJEdeP0r8YE64Dco=
+github.com/go-pg/zerochecker v0.1.1/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/segmentio/encoding v0.1.10 h1:0b8dva47cSuNQR5ZcU3d0pfi9EnPpSK6q7y5ZGEW36Q=
+github.com/segmentio/encoding v0.1.10/go.mod h1:RWhr02uzMB9gQC1x+MfYxedtmBibb9cZ6Vv9VxRSSbw=
+github.com/vmihailenco/bufpool v0.1.5 h1:mEO/biwhAgiY97yPMmAdH4PvaIu63C6uGBdfSdoMo/I=
+github.com/vmihailenco/bufpool v0.1.5/go.mod h1:fL9i/PRTuS7AELqAHwSU1Zf1c70xhkhGe/cD5ud9pJk=
+github.com/vmihailenco/msgpack/v4 v4.3.5/go.mod h1:DuaveEe48abshDmz5UBKyZ+yDugvaeFk5ayfrewUOaw=
+github.com/vmihailenco/msgpack/v4 v4.3.7 h1:Aj4eMY2qXTRfWRwHKJ2n/CWKVy15pCYN327QMJ/OUVs=
+github.com/vmihailenco/msgpack/v4 v4.3.7/go.mod h1:Ii+PksJlvFT5ZRcB/4YLAInMIp6a0WOCm0L3BU0aNG4=
+github.com/vmihailenco/tagparser v0.1.0/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190420063019-afa5a82059c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222033325-078779b8f2d8 h1:4l6HGmcZuPkox4Zl1b7SZQfYo8KIKvPe+5VG6ibuUEg=
+golang.org/x/net v0.0.0-20200222033325-078779b8f2d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=
+mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=