Merge pull request #22 from gmazzitelli/master

lab a wn update

docker/CYGNO/lab/Dockerfile

@@ -93,18 +93,30 @@ RUN pip3 install --no-cache-dir -U \
     metakernel==0.28.2 \
     tensorflow==2.6.2 \
     opencv-python==4.6.0.66 \
+    root_numpy \
+    pydot \
+    graphviz \
+    uproot \
     && pip3 install --no-cache-dir -U git+https://github.com/CYGNUS-RD/cygno.git \
     && pip3 install 'https://github.com/CYGNUS-RD/middleware/blob/master/midas/midaslib.tar.gz?raw=true'
 
 # Force default python in jupyter kernel to Python3
 COPY lab/kernel.json /usr/local/share/jupyter/kernels/python3/
 
-# Force python 3 from console
-RUN ln -nfs /usr/bin/python3 /usr/bin/python
-
 # cmake3 alias
 RUN ln -s /usr/bin/cmake3 /usr/bin/cmake
 
+# last requirments....
+RUN yum install emacs -y 
+RUN yum install screen -y
+RUN wget https://www.python.org/ftp/python/3.9.10/Python-3.9.10.tgz; \
+    tar xvf Python-3.9.10.tgz; \
+    cd Python-3.9*/; \
+    ./configure --enable-optimizations; \
+    make altinstall; \
+    cd .. ; \
+    rm -rf Python-3.9*
+
 # Link folders
 RUN mkdir -p ${JUPYTER_ROOT} \
     && ln -s /s3 ${JUPYTER_ROOT}/cloud-storage \

docker/CYGNO/lab/condor_config.local

@@ -1,5 +1,5 @@
 AUTH_SSL_CLIENT_CAFILE = /etc/pki/ca-trust/source/anchors/htcondor_ca.crt
-COLLECTOR_HOST = <IP>
 SCITOKENS_FILE = /tmp/token
-SCHEDD_HOST = <IP>
-SEC_DEFAULT_AUTHENTICATION_METHODS = SCITOKENS
+SEC_DEFAULT_AUTHENTICATION_METHODS = SCITOKENS
+COLLECTOR_HOST = 131.154.96.115.myip.cloud.infn.it:30618
+SCHEDD_HOST = 131.154.96.115.myip.cloud.infn.it

docker/CYGNO/scripts/post_script.sh

@@ -58,3 +58,5 @@ sleep 6s && nice -n 19 sts-wire https://iam.cloud.infn.it/ \
 
 # Start crond
 crond
+# automatic backup. To be check.
+# crontab -l | { cat; echo "* * * * * /bin/rsync -a --delete /jupyter-workspace/private/ /jupyter-workspace/cloud-storage/${USERNAME}/private/${IAM_CLIENT_ID} 2>&1"; } | crontab -

docker/CYGNO/tape/Dockerfile

@@ -16,4 +16,24 @@ RUN yum --setopt=tsflags=nodocs -y install epel-release yum-plugin-ovl \
     && yum clean all \
     && yum install -y ca-policy-egi-core ca-policy-lcg
 
-#RUN fetch-crl -q
+RUN  wget https://repo.data.kit.edu/data-kit-edu-centos7.repo
+
+RUN yum install -y \
+    oidc-agent \
+    && yum clean all 
+
+RUN sed -i 's/\/bin\/bin\//\/bin\//g' /usr/bin/oidc-agent-service
+
+RUN yum install -y git \
+	python3 \
+	python-pip
+
+RUN pip3 install --upgrade pip
+RUN pip3 install git+https://github.com/DODAS-TS/boto3sts
+RUN pip3 install git+https://github.com/CYGNUS-RD/cygno.git
+RUN pip3 install mysql-connector
+RUN pip2 install numpy==1.16.6
+
+RUN mkdir /root/script
+WORKDIR /root/script
+

docker/CYGNO/tape/Dockerfile.v1

@@ -0,0 +1,19 @@
+FROM cern/cc7-base:20220601-1
+
+RUN yum -y update && yum -y install \
+    wget \
+    gfal2-util \
+    gfal2-all 
+
+WORKDIR /etc/yum.repos.d
+
+RUN  wget http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo \
+     &&  wget http://repository.egi.eu/community/software/preview.repository/2.0/releases/repofiles/centos-7-x86_64.repo 
+
+RUN yum --setopt=tsflags=nodocs -y install epel-release yum-plugin-ovl \
+    && yum --setopt=tsflags=nodocs -y install epel-release yum-plugin-ovl \
+    && yum --setopt=tsflags=nodocs -y install fetch-crl wn sysstat \
+    && yum clean all \
+    && yum install -y ca-policy-egi-core ca-policy-lcg
+
+#RUN fetch-crl -q

docker/CYGNO/tape/Dockerfile.v4

@@ -0,0 +1,32 @@
+FROM gmazzitelli/cygno-tape:v1.0.1-cygno
+
+WORKDIR /etc/yum.repos.d
+
+RUN  wget https://repo.data.kit.edu/data-kit-edu-centos7.repo
+
+RUN yum install -y \
+    oidc-agent \
+    && yum clean all 
+
+RUN sed -i 's/\/bin\/bin\//\/bin\//g' /usr/bin/oidc-agent-service
+
+RUN yum install -y git \
+	python3 \
+	python-pip
+
+RUN pip3 install --upgrade pip
+RUN pip3 install git+https://github.com/DODAS-TS/boto3sts
+RUN pip3 install git+https://github.com/CYGNUS-RD/cygno.git
+RUN pip3 install mysql-connector
+RUN pip2 install numpy==1.16.6
+
+#RUN wget http://download-ib01.fedoraproject.org/pub/epel/7/aarch64/
+#RUN yum install -y gfal2-util gfal2-all
+
+RUN mkdir /root/script
+WORKDIR /root/script
+
+# RUN wget https://raw.githubusercontent.com/gmazzitelli/dodas-docker-images/master/docker/CYGNO/tape/oicd-setup.sh
+# RUN wget https://raw.githubusercontent.com/gmazzitelli/dodas-docker-images/master/docker/CYGNO/tape/cygno_s32tape.sh
+# RUN wget https://raw.githubusercontent.com/gmazzitelli/dodas-docker-images/master/docker/CYGNO/tape/s32tape.py
+# RUN chmod a+x oicd-setup.sh cygno_s32tape.sh s32tape.py

docker/CYGNO/tape/README.md

@@ -0,0 +1,88 @@
+### Usage:
+* ```docker-container up -d```
+* ```docker attach tapev4```
+* nel folder root bisogna esguire la richiesta di token con ```source oicd-setup.sh``` e poi eseguire ```./cygno_s32tape.sh```
+* [monitor spazio](https://t1metria.cr.cnaf.infn.it/d/ZArHZvEMz/storage-usage-per-experiment?orgId=18&var-exp=cygn&var-vo=CYGNO)
+
+### Tips for editing
+* ```cd <script>```
+* ```ssh -L 10000:localhost:10000 testnotebook```
+* ```docker run -d -p 10000:8888 --name editor -v "${PWD}":/home/jovyan/work/ jupyter/scipy-notebook:9e63909e0317``` 
+* e poi avere il token di accesso [guardate](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/running.html)
+* ```docker logs editor``` 
+
+
+OLD
+
+1) generare un tocken da qualunque macchina (la prima volta)
+```
+eval `oidc-agent-service use`
+oidc-gen --issuer https://iam-t1-computing.cloud.cnaf.infn.it/ \
+        --pw-cmd="echo pwd" --scope "openid profile email address phone offline_access eduperson_scoped_affiliation eduperson_entitlement" \
+         t1-tape
+```
+1.1) richimare il tocken (le volte successive)
+```
+oidc-add t1-tape
+```
+2) prendere il token con 
+```
+oidc-token t1-tape
+```
+o, se la macchina con oidc e' la stessa, configirare
+```
+TOKEN=$(oidc-token t1-tape)
+```
+3) andare contnainer che gestisce il tape e copiare, come sotto, il token e esportare la varibaile BEARER_TOKEN
+```
+TOKEN=eyJraWQiOiJyc2ExIiwiYWxnI...
+
+export BEARER_TOKEN=$TOKEN
+```
+
+* mail sistruzioni raw
+
+~~~~
+Buon pomeriggio,
+
+come richiesto, abbiamo configurato una storage area tape esposta da StoRM WebDAV dedicata all'esperimento Cygno.
+
+Per listare i file o scriverne di nuovi in questa storage area è necessario autenticarsi con un token rilasciato da un'istanza IAM che abbiamo al Tier1 condivisa fra vari esperimenti.
+
+Sarà quindi necessario
+
+- registrarsi su https://iam-t1-computing.cloud.cnaf.infn.it/login
+
+- una volta registrati, fare richiesta attraverso https://iam-t1-computing.cloud.cnaf.infn.it di membership per il gruppo "Cygno".
+
+Da command line (per esempio da ui-tier1) si può leggere/scrivere sulla storage area utilizzando tool come curl o gfal (più semplice e disponibile su ui-tier1), previa autenticazione.
+
+L'autenticazione avviene secondo il modello a token, che è spiegato nella guida https://confluence.infn.it/display/TD/Data+transfers+using+http+endpoints#Datatransfersusinghttpendpoints-TokensWebDAV
+
+Seguendo i passi descritti nella guida, sarà necessario:
+
+- utilizzare OIDC-agent per registrare un client (solo la prima volta) che rilasci i token (oidc-gen)
+
+- prendere un token (oidc-token)
+
+- fare operazioni di data management/transfer ad esempio con gfal.
+
+Abbiamo verificato che con un token rilasciato da iam-t1-computing per un utente (di User support) registrato nel gruppo Cygno, leggere, scrivere ed eliminare funzionano correttamente:
+
+$ gfal-ls davs://xfer-archive.cr.cnaf.infn.it:8443/cygno
+
+$ gfal-copy test davs://xfer-archive.cr.cnaf.infn.it:8443/cygno/prova.txt
+
+$ gfal-rm davs://xfer-archive.cr.cnaf.infn.it:8443/cygno/prova.txt
+
+Potete inoltre controllare lo stato di occupazione della storage area alla seguente pagina web:
+
+https://t1metria.cr.cnaf.infn.it/d/ZArHZvEMz/storage-usage-per-experiment?orgId=18&var-exp=cygn&var-vo=CYGNO
+
+Ovviamente, potete sempre contattare User support per problemi o difficoltà.
+
+Cordiali saluti,
+
+   Andrea
+~~~~~
+

docker/CYGNO/tape/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3"
+services:
+  app:
+    image: gmazzitelli/cygno-tape:v1.0.5-cygno
+    container_name: tapev5
+    stdin_open: true # docker run -i
+    tty: true        # docker run -t
+    volumes:
+      - "./script/:/root/script"

docker/CYGNO/tape/script/.ipynb_checkpoints/0s32tape-checkpoint.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+import gfal2
+import logging
+import optparse
+import sys
+import os
+from optparse import OptionParser
+import cygno as cy
+
+def event_callback(event):
+    #print event
+    print("[%s] %s %s %s" % (event.timestamp, event.domain, event.stage, event.description))
+
+def monitor_callback(src, dst, average, instant, transferred, elapsed):
+    print("[%4d] %.2fMB (%.2fKB/s)\r" % (elapsed, transferred / 1048576, average / 1024)),
+    sys.stdout.flush()
+
+def copy_s32tape(filename, backet="cygno-data", intag="LNGS", outtag=""):   
+    # filename = "run02308.mid.gz"
+    source =  "https://s3.cloud.infn.it/v1/AUTH_2ebf769785574195bde2ff418deac08a/{:s}/{:s}/{:s}".format(backet, intag, filename)
+    if outtag != "":
+        dest_p = "{:s}/{:s}".format(outtag, filename)
+    else:
+        dest_p = filename
+    dest   =  "davs://xfer-archive.cr.cnaf.infn.it:8443/cygno/{:s}".format(dest_p)
+    dest_cred = os.environ['TAPE_TOKEN']
+
+# ADD here the T1 tape token
+# dest_cred = ""
+
+    print("Source:      %s" % source)
+    print("Destination: %s" % dest)
+    print("Destination Token: %s" % dest_cred)
+
+# Instantiate gfal2
+    ctx = gfal2.creat_context()
+
+# Set transfer parameters
+    params = ctx.transfer_parameters()
+    params.event_callback   = event_callback
+    params.monitor_callback = monitor_callback
+
+        # to enable if needed [basic examples]
+#        gfal2.set_verbose(gfal2.verbose_level.debug)
+    params.overwrite = True
+#        params.checksum_check = True
+
+# not necessarily needed. 
+# current cygno data are www readable 
+#       s_cred = ctx.cred_new("BEARER",source_cred)
+#       ctx.cred_set(source,s_cred)
+#       print("Source credentials: %s" % source_cred)
+
+# writing on tape at T1 requires authN/Z
+    d_cred = ctx.cred_new("BEARER",dest_cred)
+    ctx.cred_set(dest,d_cred)
+    print("Destination credentials: %s" % dest_cred)
+
+# Five minutes timeout
+    params.timeout = 300
+
+# Do actual copy using different protocols for source and destination 
+    try:
+        r = ctx.filecopy(params, source, dest)
+        print("Copy succeeded!")
+    except Exception as e:
+        print("Copy failed: %s" % str(e))
+        sys.exit(1)   
+
+def main(backet, options, options, options):
+    print(backet, options, options, options)
+
+if __name__ == '__main__':
+    cygno_backet_list = ["cygnus", "cygno-data", "cygno-sim", "cygno-analysis"]
+    #
+    parser = OptionParser(usage='usage: %prog\t [-tsv] [ls backet]\n\t\t\t [put backet filename]\n\t\t\t [[get backet filein] fileout]\n\t\t\t [rm backet fileneme]\nAvailable Backet: '+str(cygno_backet_list)+\
+        "\n recall to run comman: \n eval \`oidc-agent\` \n oidc-gen --reauthenticate --flow device infncloud-iam")
+    parser.add_option('-b','--backet', dest='backet', type='string', default='cygno-data', help='backet on s3;');
+    parser.add_option('-t','--intag', dest='intag', type='string', default='LNGS', help='tag on s3;');
+    parser.add_option('-o','--outtaag', dest='intag', type='outtaag', default='LNGS', help='out tag on tape;');
+    parser.add_option('-s','--session', dest='session', type='string', default='infncloud-iam', help='token profile [infncloud-iam];');
+    parser.add_option('-v','--verbose', dest='verbose', action="store_true", default=False, help='verbose output;');
+    (options, args) = parser.parse_args()
+    #
+    if options.verbose: 
+        print(">> resquested arguments:", args)
+        print(">> resquested options:", options)
+    #       
+    # if len(args) < 2:
+    #     parser.error("incorrect number of arguments")
+    # if not (args[1] in cygno_backet_list):
+    #     error = "backet not availabe in cygno repo: "+str(cygno_backet_list)
+    #     parser.error(error)
+
+
+    main(options.backet, options.intag, options.outtag, options.session)
+