'+(i?e:gn(e,!0))+"\n":""+(i?e:gn(e,!0))+"\n"+e+"\n"},e.prototype.html=function(e){return e},e.prototype.heading=function(e,n,i,o){return this.options.headerIds?"
\n":"
\n"},e.prototype.list=function(e,n,i){var o=n?"ol":"ul";return"<"+o+(n&&1!==i?' start="'+i+'"':"")+">\n"+e+"\n"},e.prototype.listitem=function(e){return"
"+e+"
\n"},e.prototype.table=function(e,n){return""+e+""},e.prototype.br=function(){return this.options.xhtml?"":"
"},e.prototype.del=function(e){return"
"+wn(e.message+"",!0)+"";throw e}}xn.options=xn.setOptions=function(e){return bn(xn.defaults,e),yn(xn.defaults),xn},xn.getDefaults=Me,xn.defaults=we,xn.use=function(a){var n,e=bn({},a);if(a.renderer){var i,r=xn.defaults.renderer||new sn;for(i in a.renderer)!function(o){var t=r[o];r[o]=function(){for(var e=[],n=arguments.length;n--;)e[n]=arguments[n];var i=a.renderer[o].apply(r,e);return i=!1===i?t.apply(r,e):i}}(i);e.renderer=r}if(a.tokenizer){var t,c=xn.defaults.tokenizer||new nn;for(t in a.tokenizer)!function(){var o=c[t];c[t]=function(){for(var e=[],n=arguments.length;n--;)e[n]=arguments[n];var i=a.tokenizer[t].apply(c,e);return i=!1===i?o.apply(c,e):i}}();e.tokenizer=c}a.walkTokens&&(n=xn.defaults.walkTokens,e.walkTokens=function(e){a.walkTokens(e),n&&n(e)}),xn.setOptions(e)},xn.walkTokens=function(e,n){for(var i=0,o=e;i
"+wn(e.message+"",!0)+"";throw e}},xn.Parser=mn,xn.parser=mn.parse,xn.Renderer=sn,xn.TextRenderer=ln,xn.Lexer=fn,xn.lexer=fn.lex,xn.Tokenizer=nn,xn.Slugger=vn;var Sn=xn.parse=xn;function An(e,i){if(void 0===i&&(i=''),!e||!e.length)return"";var o="";return e.forEach(function(e){var n=e.title.replace(/(<([^>]+)>)/g,"");o+='
'+n.slice(5).trim()+"
"}function zn(e,o){var t=[],a={};return e.forEach(function(e){var n=e.level||1,i=n-1;o
':i;var i,o}).replace(/__colon__/g,":")}function Ln(e){var o={};return{str:e=(e=void 0===e?"":e)&&e.replace(/^('|")/,"").replace(/('|")$/,"").replace(/(?:^|\s):([\w-]+:?)=?([\w-%]+)?/g,function(e,n,i){return-1===n.indexOf(":")?(o[n]=i&&i.replace(/"/g,"")||!0,""):e}).trim(),config:o}}function On(e){return(e=void 0===e?"":e).replace(/(<\/?a.*?>)/gi,"")}var qn,Pn=be(function(e){var u,f,p,d,n,g=function(u){var i=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,n=0,e={},R={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof T?new T(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=r.reach);m+=_.value.length,_=_.next){var b=_.value;if(i.length>n.length)return;if(!(b instanceof T)){var k,w=1;if(l){if(!(k=C(h,m,n,s))||k.index>=n.length)break;var y=k.index,x=k.index+k[0].length,S=m;for(S+=_.value.length;S<=y;)_=_.next,S+=_.value.length;if(S-=_.value.length,m=S,_.value instanceof T)continue;for(var A=_;A!==i.tail&&(S'+Pn.highlight(e.replace(/@DOCSIFY_QM@/g,"`"),i,n)+""+e+"
";return e},t.image=(o=(i={renderer:e,contentBase:o,router:a}).renderer,p=i.contentBase,d=i.router,o.image=function(e,n,i){var o=e,t=[],a=Ln(n),r=a.str,a=a.config;return n=r,a["no-zoom"]&&t.push("data-no-zoom"),n&&t.push('title="'+n+'"'),a.size&&(n=(r=a.size.split("x"))[0],(r=r[1])?t.push('width="'+n+'" height="'+r+'"'):t.push('width="'+n+'"')),a.class&&t.push('class="'+a.class+'"'),a.id&&t.push('id="'+a.id+'"'),R(e)||(o=q(p,C(d.getCurrentPath()),e)),0\x3c!--cover--\x3e
'+e.title+"
\n"+e.content+"
\n\n"}),t.classList.add("show"),a.classList.add("show"),t.innerHTML=r||''+c+"
",s.hideOtherSidebarContent&&(i&&i.classList.add("hide"),n&&n.classList.add("hide"))}function l(e){s=e}function h(e,n){var t,a,i=n.router.parse().query.s;l(e),Docsify.dom.style("\n.sidebar {\n padding-top: 0;\n}\n\n.search {\n margin-bottom: 20px;\n padding: 6px;\n border-bottom: 1px solid #eee;\n}\n\n.search .input-wrap {\n display: flex;\n align-items: center;\n}\n\n.search .results-panel {\n display: none;\n}\n\n.search .results-panel.show {\n display: block;\n}\n\n.search input {\n outline: none;\n border: none;\n width: 100%;\n padding: 0.6em 7px;\n font-size: inherit;\n border: 1px solid transparent;\n}\n\n.search input:focus {\n box-shadow: 0 0 5px var(--theme-color, #42b983);\n border: 1px solid var(--theme-color, #42b983);\n}\n\n.search input::-webkit-search-decoration,\n.search input::-webkit-search-cancel-button,\n.search input {\n -webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n}\n\n.search input::-ms-clear {\n display: none;\n height: 0;\n width: 0;\n}\n\n.search .clear-button {\n cursor: pointer;\n width: 36px;\n text-align: right;\n display: none;\n}\n\n.search .clear-button.show {\n display: block;\n}\n\n.search .clear-button svg {\n transform: scale(.5);\n}\n\n.search h2 {\n font-size: 17px;\n margin: 10px 0;\n}\n\n.search a {\n text-decoration: none;\n color: inherit;\n}\n\n.search .matching-post {\n border-bottom: 1px solid #eee;\n}\n\n.search .matching-post:last-child {\n border-bottom: 0;\n}\n\n.search p {\n font-size: 14px;\n overflow: hidden;\n text-overflow: ellipsis;\n display: -webkit-box;\n -webkit-line-clamp: 2;\n -webkit-box-orient: vertical;\n}\n\n.search p.empty {\n text-align: center;\n}\n\n.app-name.hide, .sidebar-nav.hide {\n display: none;\n}"),function(e){void 0===e&&(e="");var n=Docsify.dom.create("div",'\n \n \n
\n \n '),e=Docsify.dom.find("aside");Docsify.dom.toggleClass(n,"search"),Docsify.dom.before(e,n)}(i),n=Docsify.dom.find("div.search"),a=Docsify.dom.find(n,"input"),e=Docsify.dom.find(n,".input-wrap"),Docsify.dom.on(n,"click",function(e){return-1===["A","H2","P","EM"].indexOf(e.target.tagName)&&e.stopPropagation()}),Docsify.dom.on(a,"input",function(n){clearTimeout(t),t=setTimeout(function(e){return d(n.target.value.trim())},100)}),Docsify.dom.on(e,"click",function(e){"INPUT"!==e.target.tagName&&(a.value="",d())}),i&&setTimeout(function(e){return d(i)},500)}function x(e,n){var t,a,i,r,o;l(e),t=e.placeholder,a=n.route.path,(r=Docsify.dom.getNode('.search input[type="search"]'))&&("string"==typeof t?r.placeholder=t:(i=Object.keys(t).filter(function(e){return-1MOLGENIS Armadillo
+A simple, user-friendly, open source solution for implementing federated analysis.+Armadillo is a data portal that allows data stewards to share datasets on a server, +and researchers to analyse these data and those shared on other servers using +the DataSHIELD analysis tools.
+{.intro} +> Why is it called Armadillo?
+> - The word Armadillo is Spanish and means "little armored one". MOLGENIS Armadillo is a compact solution that implements +> the DataSHIELD infrastructure. \ No newline at end of file diff --git a/docs/ops/configuring.md b/docs/ops/configuring.md deleted file mode 100644 index 55890b1b1..000000000 --- a/docs/ops/configuring.md +++ /dev/null @@ -1,23 +0,0 @@ -# Using Armadillo -Armadillo has three main screens to manage projects, user access and DataSHIELD profiles: - -### Create data access projects -Data stewards can use the Armadillo web user interface or [MolgenisArmadillo R client](https://molgenis.github.io/molgenis-r-armadillo) -to create 'projects' and upload their data into those. Data tables need to be in parquet format that supports fast selections of the columns -(variables) you need for analysis. Other files can be configured as 'resources'. - -### Manage user access -Data stewards can use the permission screen to give email adresses access to the data. Everybody signs in via single sign on using an OIDC central -authentication server such as KeyCloack or Fusion auth that federates to authentication systems of -connected institutions, ideally using a federated AAI such as LifeScience AAI. - -### Configure DataSHIELD profiles -To analyse data, users must choose a datashield profile. Armadillo owners can use the web user interface to configure new profiles. Assuming you -installed docker you can also start/stop these images. Alternatively you can use docker-compose for that. We -recommend selecting one of the -[DataSHIELD standard profiles](https://www.datashield.org/help/standard-profiles-and-plaforms). - -### End users can use Armadillo as any other DataSHIELD server -A researcher connects from an [R client](https://molgenis.github.io/molgenis-r-datashield) to one or multiple Armadillo servers. The data is -loaded into an R session on the Armadillo server specifically created for the researcher. Analysis requests are sent to the R session on each Armadillo server. -There the analysis is performed and aggregated results are sent back to the client. diff --git a/docs/ops/install/install_apache.md b/docs/ops/install/install_apache.md deleted file mode 100644 index 11379e7d1..000000000 --- a/docs/ops/install/install_apache.md +++ /dev/null @@ -1,32 +0,0 @@ -# Apache - -It is possible to run Molgenis Armadillo using Apache, however we do not provide support with this configuration. - -## Encoding - -Apache requires some additional configuration to get the `/storage/projects/{project}/objects/{object}` to work. When this endpoint doesn't work: - -- tables cannot be assigned -- subsets cannot be created -- resources cannot be used. - -## Tell Armadillo about https - -We need to tell Armadillo server how to building URLs. - -## Changes to your site-enabled configuration - -Your configuration probably should look like this. - -```conf -ProxyPreserveHost On - -ProxyPass / http://localhost:8080/ nocanon - -AllowEncodedSlashes On - -RequestHeader set X-Forwarded-Proto https -RequestHeader set X-Forwarded-Port 443 -``` - -After setting this don't forget to restart Apache. diff --git a/docs/ops/install/install_docker.md b/docs/ops/install/install_docker.md deleted file mode 100644 index 6f3e476f9..000000000 --- a/docs/ops/install/install_docker.md +++ /dev/null @@ -1,10 +0,0 @@ -## Run Armadillo via docker compose -For testing without having to installing Java you can run using docker: - -1. Install [docker-compose](https://docs.docker.com/compose/install/) -2. Download this [docker-compose.yml](docker-compose.yml). -3. Execute ```docker-compose up``` -4. Once it says 'Started' go to http://localhost:8080 to see your Armadillo running. - -The command must run in the same directory as the downloaded docker file. We made docker available via 'docker.sock' so we can start/stop DataSHIELD profiles. Alternatively you must include the datashield profiles into this docker-compose. You can override all application.yaml settings via environment variables -(see commented code in docker-compose file). diff --git a/docs/ops/install/install_java.md b/docs/ops/install/install_java.md deleted file mode 100644 index 1c00246ea..000000000 --- a/docs/ops/install/install_java.md +++ /dev/null @@ -1,13 +0,0 @@ -## Run Armadillo using java commandline -Software developers often run Armadillo as java jar file: - -1. Install Java and Docker (for the DataSHIELD profiles) -2. Download Armadillo jar file from [releases](https://github.com/molgenis/molgenis-service-armadillo/releases), for example: -[molgenis-armadillo-3.3.0.jar](https://github.com/molgenis/molgenis-service-armadillo/releases/download/V3.3.0/) -3. Run armadillo using ```java -jar molgenis-armadillo-3.3.0.jar``` -4. Go to http://localhost:8080 to see your Armadillo running. - -Default Armadillo will start with only 'basic-auth' and user 'admin' with password 'admin'. You can enable 'oidc' for connecting more users. You can change -by providing and editing [application.yaml](application.template.yml) file -in your working directory and then run command above again. - diff --git a/docs/overrides/partials/copyright.html b/docs/overrides/partials/copyright.html new file mode 100644 index 000000000..440b79c14 --- /dev/null +++ b/docs/overrides/partials/copyright.html @@ -0,0 +1,13 @@ +
+ Please cite Cadman et al (2024) when using MOLGENIS Armadillo.
+
diff --git a/docs/pages/about_us.md b/docs/pages/about_us.md
new file mode 100644
index 000000000..9a2186f2e
--- /dev/null
+++ b/docs/pages/about_us.md
@@ -0,0 +1,7 @@
+# About us
+Armadillo is part of the [MOLGENIS community](https://molgenis.org/communities.html). We are an
+[enthusiastic group](https://molgenis.org/team.html) of datamanagers, systemoperator and developers. We have a large
+number of national and international partners and value research, data-sharing, collaboration and open source software.
+
+We are also involved in the [DataSHIELD community](https://wiki.datashield.org/). This community is very passionate when
+it comes to federated analyses and sharing data securely. They are very welcoming to anyone willing to participate.
\ No newline at end of file
diff --git a/docs/pages/advanced_usage.md b/docs/pages/advanced_usage.md
new file mode 100644
index 000000000..9b3e398a5
--- /dev/null
+++ b/docs/pages/advanced_usage.md
@@ -0,0 +1,6 @@
+# Advanced Usage
+API usage
+
+monitoring
+
+Logs
\ No newline at end of file
diff --git a/docs/pages/basic_concepts.md b/docs/pages/basic_concepts.md
new file mode 100644
index 000000000..94880d4bb
--- /dev/null
+++ b/docs/pages/basic_concepts.md
@@ -0,0 +1,42 @@
+# Basic concepts
+[DataSHIELD](https://datashield.org/) is an R-based software solution for federated analysis - the remote analysis of multiple data sources.
+It allows for sophisticated analyses without the user being able to view or copy individual level data. Instead, only non-disclosive
+summary statistics are returned. This makes it an effective solution for secure data science collaborations.
+
+In order to use DataSHIELD, additional software is required to store data and manage user interaction. There are currently two solutions for doing this: Armadillo and [Opal](https://opaldoc.obiba.org/), which can be used compatibly within the same network. Below is an example of a simple setup:
+
+### Example setup 1
+{ width="600" }
+/// caption
+A user writes analysis commands in R, using client-side DataSHIELD R packages. The commands are sent to the biobank
+servers. For Armadillo, the communication between the client and the server is handled by the R packages [DSI](https://github.com/datashield/DSI) and [DSMolgenisArmadillo](https://molgenis.github.io/molgenis-r-datashield/), whilst the data storage and execution of commands on the server is handled by [ArmadilloService](https://molgenis.github.io/molgenis-service-armadillo/). The non-disclosive summary statistics are then returned to the user.
+///
+
+### Example setup 2
+An alternative setup involves the user first connecting to a Central Analysis Server (CAS), which is an online R studio environment:
+
+
+/// caption
+Once logged in to the CAS, users write their code as if they were running RStudio locally. The advantage of this setup is that Biobank servers can be configured so that they are blocked off from the rest of the internet by a firewall and can only be accessed from the CAS. This provices an additional layer of data protection. It also benefits users, as all required DataSHIELD R packages can be pre-installed thus removing the needs for users to set up their R environment.
+///
+
+### Resources
+An additional optional feature of DataSHIELD is the ability to host files elsewhere (e.g. computer clusters) and link them to
+the data of armadillo or opal servers. This is impletmented using the [resourcer](https://github.com/obiba/resourcer) package. External resources can
+be used alongside data stored in armadillo itself, and resources may be hosted in different locations and formats.
+
+### DataSHIELD packages and their use
+Finally, here is a brief summary of the core Armadillo and DataSHIELD packages described in this documentation.
+
+| **Name** | **Type of application** | **User type** | **Description** | Sources |
+|---------------------:|------------------------------------------------------------|-----------------|-------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
+| armadilloService | :material-language-java: Java | - | The software that facilitates hosting and analysing data using DataSHIELD. | [:simple-github:](https://github.com/molgenis/molgenis-service-armadillo) |
+| DSI | :simple-r: R package | - | This package defines the API that is to be implemented by DataSHIELD compliant data repositories. | [:simple-github:](https://github.com/datashield/DSI) |
+| DSMolgenisArmadillo | :simple-r: R package | Researchers | To communicate between the client-side packages and armadilloService to perform analysis. | [:simple-github:](https://github.com/molgenis/molgenis-R-datashield) [:simple-r:](https://cran.r-project.org/web/packages/DSMolgenisArmadillo/index.html) |
+| MolgenisArmadillo | :simple-r: R package | Datamanager | To upload data and manage projects in Armadillo in R. | [:simple-github:](https://github.com/molgenis/molgenis-R-armadillo) [:simple-r:](https://cran.r-project.org/web/packages/MolgenisArmadillo/index.html) |
+| User Interface| :simple-javascript: Javascript/
+ Documentation made with
+
+ Material for MkDocs
+
+
+
+:material-vuejs: VueJS | Datamanager | To manage users, data, configurations and view logs. | [:simple-github:](https://github.com/molgenis/molgenis-service-armadillo/tree/master/ui) | +| dsUpload | :simple-r: R package | Datamanager | To upload data according to a specific format into Armadillo/Opal using R. | [:simple-github:](https://github.com/lifecycle-project/ds-upload) | +| dsBaseClient | :simple-r: R package | Researchers | Core DataSHIELD client-side R package required for executing basic DataSHIELD commands. | [:simple-github:](https://github.com/datashield/dsBaseClient) | +| dsTidyverseClient | :simple-r: R package | Researchers | Client-side R package which implements efficient data-manipulation using selected Tidyverse functions. | [:simple-github:](https://github.com/molgenis/ds-tidyverse-client) [:simple-r:](https://cran.r-project.org/web/packages/dsTidyverseClient/index.html) | + + diff --git a/docs/pages/basic_usage/armadillo_ui.md b/docs/pages/basic_usage/armadillo_ui.md new file mode 100644 index 000000000..67e4265ae --- /dev/null +++ b/docs/pages/basic_usage/armadillo_ui.md @@ -0,0 +1,172 @@ +# Armadillo UI + +Relevant for: :material-file-table:{title="Data Managers"} + +Armadillo features a user interface, or UI for short. + +## Login + + + +To login to the UI, select the **Institute account (oath2)** button and login using the institute or organisation login screen you will be redirected to. + +### Superuser + +You need to have admin or superuser permissions if you want to add projects, users or profiles. This means you need to be granted permission in order to be able to use the UI. If you don't have correct permissions, you will receive the +following error: + + + +If you receive this error, contact someone in your institute that is already able to login without an error, or [contact MOLGENIS Support](../contact.md). + +To grant a user superuser permissions simply search for that user in the `Users` tab of the UI, and tick the _admin_ checkbox for that user: + + + +## Projects + +Once you're logged in, you will be redirected to the `Projects` page. On this page you can add and edit projects. + +You can add users to projects and navigate to the "project-editor"-view and search through the projects using the search bar on the top right. + + + +### Editing projects + +To edit your project, click on the edit button in front of the project you want to edit: {width="15"}. + +The row will be opened in edit mode: + + + +The edit mode can be recognized by its blue background color and you have the option to add new users to your project by clicking on the + button {width="15"} of the users column. Then, you can either select an existing user from the dropdown, or add the email address of a new user. + + + +In case of adding a user in this screen, a warning will be shown to prevent email addresses with typographical errors from being added to your system. For example, if you have the address `j.doe@example.com` in your users table but attempt to add `j.die@example.com` a warning message will be displayed asking if you want to add a new user. + + + +It is not possible to edit the name of your project; this is intentional in order to ensure tables, resources, users, and permissions are transferred to the new project name correctly. + +Click on the checkmark {width="25"} to save the edited row and the X {width="25"} to cancel. + +Be careful, if you do cancel your changes will be lost. + +### Adding projects + +To add a new project, click on the + button {width="25"} on top of the table. If you click this button, an empty row will be opened in edit mode. + + + +Click on the checkmark {width="25"} to save the edited row and the X {width="25"} to cancel. + +Be careful, if you do cancel your changes will be lost. + +## Project explorer + +If you click on the icon {width="40"} next to the project name, you will be directed to the `project editor`. In this screen you can upload and preview data in projects. To go back to your `projects` page, +press the back button {width="25"}. + + + +If you click on a folder, it will open. + + + +Here you can upload files to that folder, or click on the tables (files) to preview their contents. + + + +To upload files, either drag a file from your file browser to the file upload area, or click the area and select the +file. + + + +After selecting the file, click on _upload_ to upload it. This usually only takes a few seconds but will take longer in your uploading a large file. + +It is also possible to create new folders. To do so, click the "add folder" button {width="30"} just below the project name. An input dialog will be presented: + + + +Fill in the name you want to use and click on the checkmark button {width="25"}. Please keep in mind that the folder will only be saved if you put data into it. Select the new folder to select files to upload. + +### Resources + +All file types can be uploaded into Armadillo, however previews will only be available for `.parquet` files. Other files that can be uploaded are treated as _resources_. Resource filetypes usually are `.rda` files or `.Rdata` files. + +To be able to use these resources as a researcher, first an `.rds` file must be generated. How to create these files, is described [here](https://molgenis.github.io/molgenis-r-armadillo/articles/create_resources.html). + +The URL of your resources should consist of: + +```r +{your url}/storage/projects/{project name}/objects/{name of the folder}%2F{the resource file} +``` + +Here is an example, with some example parameters: + +```r +url = "https://armadillo3.demo.molgenis.net" +project = "omics" +folder = "ewas" +file = "gse66351_1.rda" +``` + +Which results in the following url: + +```r +https://armadillo3.demo.molgenis.net/storage/projects/omics/objects/ewas%2Fgse66351_1.rda +``` + +## Users + +The `Users` page works just as the `Projects` page. You can search users by entering (a part of) the email address or name of the user into the search box: + + + +### Editing users + +Users can be edited, **except** for their email addresses. This is because when user's email address changes, that user is possibly no longer working for the same institution and therefore might not be allowed to access the data anymore. + + + +In edit mode, the row will turn blue. Projects can be added by clicking on the + icon {width="15"}in the projects column. + +{width="350"} + +You can add a new project by typing it and clicking the checkmark button {width="25"}. You will be prompted with a warning message, asking you to confirm if you want to add a new project. + +Alternatively, you can select an existing project by using the search box or scrolling through the presented list. + +### Adding users + +By clicking on the plus button {width="25"} on the top of the table, a new user can be added. The row with the new user will turn blue in edit mode. + +{width="650"} + +Users can be added before they have logged in previously. These users can be added to projects, which will grant them permission to use the data from those projects upon their first login. Researchers should not be set as admin. + +## Analysis Profiles + + + +Since the release of Armadillo 3.0.0, it is possible to create and manage analysis profiles in the user interface, rather than asking system administrators to manage these profiles. You can start {width="35"} and stop {width="35"} profiles. + +When you start a profile for the first time, it will take a bit longer to load because the profile needs to be downloaded and installed before it can be started. + + + +If you switch to another screen whilst either starting or stopping a profile, the profiles page will no longer show the loading information. It is however still loading, and when it's done, if you reload the page, you will see that your profile started or stopped successfully. + +As in the other screens, you can add profiles with the add-button {width="25"}. + + + +By default, some fields will be set. Please update them to install the correct profile. + +Possible images can be found on [dockerhub](https://hub.docker.com/search?q=datashield%2Farmadillo-rserver). We recommend selecting one of the +[DataSHIELD standard profiles](https://wiki.datashield.org/en/opmanag/standard-profiles-and-platforms). The image name of those profiles can be found on the dockerhub link above. + +Although the default `port` setting should find an available port, please keep in mind that the port has to be unique, otherwise you cannot start your profile and will receive an error message. + +R packages can be whitelisted by adding them to the `package whitelist` column so researchers can use them. 'Whitelisting' a package allows analysts to use it. If you want to whitelist a package, you need to make sure it is installed on the image you selected. Additionally, it is possible to blacklist certain R functions in the `blacklist function` column. This can be useful if certain functions are not allowed to be used on certain data or within certain cohorts. diff --git a/docs/pages/basic_usage/auth.md b/docs/pages/basic_usage/auth.md new file mode 100644 index 000000000..21a59cdd5 --- /dev/null +++ b/docs/pages/basic_usage/auth.md @@ -0,0 +1,5 @@ +# Authentication server + +Relevant for: :material-server:{title="System Operator"} + +Authentication server: [https://lifecycle-auth.molgenis.org/](https://lifecycle-auth.molgenis.org/) \ No newline at end of file diff --git a/docs/pages/basic_usage/central_analysis_server.md b/docs/pages/basic_usage/central_analysis_server.md new file mode 100644 index 000000000..f9b810187 --- /dev/null +++ b/docs/pages/basic_usage/central_analysis_server.md @@ -0,0 +1,7 @@ +# Central Analysis Server (CAS) + +Relevant for: :material-layers-search:{title="Researchers"} + +Most networks use the analysis environment instead of their locally installed R/Rstudio environment. The analysis environment is centralized. You will be using a web based RStudio which is available here: [https://lifecycle.analysis.molgenis.org/](https://lifecycle.analysis.molgenis.org/) + +The benefit of using a CAS server is a additional layer of security, only traffic from the CAS server is accepted by the Armadillo server. diff --git a/docs/pages/basic_usage/ds_molgenis_armadillo.md b/docs/pages/basic_usage/ds_molgenis_armadillo.md new file mode 100644 index 000000000..c52a90b7b --- /dev/null +++ b/docs/pages/basic_usage/ds_molgenis_armadillo.md @@ -0,0 +1,70 @@ +# DSMolgenisArmadillo + +Relevant for: :material-layers-search:{title="Researchers"} + +Use DSMolgenisArmadillo to analyse data shared in MOLGENIS Armadillo servers using DataSHIELD. DataSHIELD allows execution of a subset of analysis methods available in R. Methods such as: + +`ds.mean()` `ds.glm()` `ds.lmerSLMA()` + +For more detailed information please visit [DSMolgenisArmadillo](https://molgenis.github.io/molgenis-r-datashield/) and additionally check the documentation: [https://cran.datashield.org/](https://cran.datashield.org/). + +=== "Prerequisite" + + ???+ example + + ```r linenums="1" title="Prerequisite.r" + install.packages("DSI") + install.packages("DSMolgenisArmadillo") + install.packages("dsBaseClient", repos = c("http://cran.datashield.org", "https://cloud.r-project.org/"), dependencies = TRUE) + + library(dsBaseClient) + library(DSMolgenisArmadillo) + ``` +=== "Basic usage" + ???+ example + + ```r linenums="1" title="Analyse_data_subset_DSMolgenisArmadillo.r" + library(dsBaseClient) + library(DSMolgenisArmadillo) + + # specify server url + armadillo_url <- "https://armadillo.test.molgenis.org" + + # get token from central authentication server + token <- armadillo.get_token(armadillo_url) + + + # build the login dataframe + builder <- DSI::newDSLoginBuilder() + builder$append( + server = "armadillo", + url = armadillo_url, + token = token, + table = "workshop1/2_1-core-1_0/nonrep", + driver = "ArmadilloDriver" + ) + + # create loginframe + logindata <- builder$build() + + # login into server + conns <- datashield.login( + logins = logindata, + symbol = "core_nonrep", + variables = c("coh_country"), + assign = TRUE + ) + + # calculate the mean + ds.mean("core_nonrep$coh_country", datasources = conns) + + ds.histogram(x = "core_nonrep$coh_country", datasources = conns) + ``` + +!!! info + A researcher only has access to data if he/she has been granted access to this data (typically) by a data manager or administrator of a cohorts Armadillo server. + +!!! note + The example files used in the examples above can be found here: [nonrep.parquet](../../data/nonrep.parquet), [yearlyrep.parquet](../../data/yearlyrep.parquet) + + Load these `parquet` files in R and see which variables you can call. diff --git a/docs/pages/basic_usage/dslite.md b/docs/pages/basic_usage/dslite.md new file mode 100644 index 000000000..af4711613 --- /dev/null +++ b/docs/pages/basic_usage/dslite.md @@ -0,0 +1,10 @@ +# DSLite + +Relevant for: :fontawesome-solid-laptop-code:{title="Developers"} + +DataSHIELD Lite ([DSLite](https://github.com/datashield/DSLite)) is a serverless [DataSHIELD Interface (DSI)](https://datashield.github.io/DSI/) implementation which purpose is to mimic the behavior of a distant (virtualized or barebone) data repository server like Armadillo or Opal. The datasets that are being analyzed are fully accessible in the local environment. The DataSHIELD configuration (set of allowed aggregation and assignment functions) is discovered at runtime by inspecting the DataSHIELD server-side R packages installed locally. This configuration can also be amended or provided explicitly. + +DSLite can be used to: + +- speed up development and testing cycle when developing new DataSHIELD functions (both at server and client side): no need to deploy a data repository infrastructure. +- allow DataSHIELD analysis with combined datasets, some of them being accessible remotely in secure data repositories, others being privately accessible (in a governmental institution for instance). diff --git a/docs/pages/basic_usage/dsupload_dsdictionaries.md b/docs/pages/basic_usage/dsupload_dsdictionaries.md new file mode 100644 index 000000000..c4a072b3f --- /dev/null +++ b/docs/pages/basic_usage/dsupload_dsdictionaries.md @@ -0,0 +1,10 @@ +# DsUpload & DsDictionaries + +Relevant for: :material-file-table:{title="Data Managers"} + +DataSHIELD upload tools ([DsUpload](https://lifecycle-project.github.io/ds-upload/)) s a collections of tools used to upload data into DataSHIELD backends like Armadillo and Opal. It aids data managers in the initial stages of uploading data to DataSHIELD backends. + +DsUpload depends on [DsDictionaries](https://github.com/lifecycle-project/ds-dictionaries) which holds the [harmonised data dictionaries](https://github.com/lifecycle-project/ds-dictionaries/tree/master/dictionaries) of multiple projects like [Lifecycle](https://data-catalogue.molgeniscloud.org/catalogue/ssr-catalogue/LifeCycle) and [Athlete](https://data-catalogue.molgeniscloud.org/catalogue/ssr-catalogue/ATHLETE). + +!!! info + Visit MOLGENIS [European Health Research Data and Sample Catalogue](https://data-catalogue.molgeniscloud.org/) which is used by projects like Lifecycle and Athlete to harmonise their variables! diff --git a/docs/pages/basic_usage/index.md b/docs/pages/basic_usage/index.md new file mode 100644 index 000000000..e15dd166f --- /dev/null +++ b/docs/pages/basic_usage/index.md @@ -0,0 +1,22 @@ +# Basic Usage + +Depending on your [role](../quick_start.md#quick-start) you have different options to interact with MOLGENIS Armadillo. + +=== ":material-file-table: Data Manager" + + A [data manager](../quick_start.md#data-manager) will typically interact with Armadillo in multiple ways: + + - Using the [Armadillo User Interface (UI)](../basic_usage/armadillo_ui.md) + - Using a R client [MolgenisArmadillo](../basic_usage/molgenis_armadillo.md) + - Using [dsUpload](../basic_usage/dsupload_dsdictionaries.md) (R) to upload data with the help of predefined harmonised data dictionaries ([ds-dictionaries](../basic_usage/dsupload_dsdictionaries.md)). + +=== ":material-server: System Operator" + [System operator](../quick_start.md#system-operator): [Authentication server](../basic_usage/auth.md) + +=== ":material-layers-search: Researcher" + + A [researcher](../quick_start.md#researcher) will connect to a Armadillo server with a R client [DSMolgenisArmadillo](../basic_usage/ds_molgenis_armadillo.md), this client allows for execution of DataSHIELD functions. Depending on the way your cohort has been setup you might use the R client through the [Central Analysis server (CAS)](../basic_usage/central_analysis_server.md). In both cases the researcher has installed R and Rstudio to edit and run their analysis. + +=== ":fontawesome-solid-laptop-code: Developer" + + A [developer](../quick_start.md#developer) can use DataSHIELD Lite ([DSLite](../basic_usage/dslite.md)) as a serverless [DataSHIELD Interface (DSI)](https://datashield.github.io/DSI/) implementation which purpose is to mimic the behavior of a distant (virtualized or barebone) data repository server like Armadillo or Opal. diff --git a/docs/pages/basic_usage/molgenis_armadillo.md b/docs/pages/basic_usage/molgenis_armadillo.md new file mode 100644 index 000000000..da2a2a458 --- /dev/null +++ b/docs/pages/basic_usage/molgenis_armadillo.md @@ -0,0 +1,252 @@ +# MolgenisArmadillo + +Relevant for: :material-file-table:{title="Data Managers"} + +MolgenisArmadillo is a R library that can be used by data managers to share datasets on a MOLGENIS Armadillo server. Researchers can then analyse these datasets and datasets shared on other servers using DataSHIELD. Researchers will only be able to access aggregate information and cannot see individual rows. + +For more detailed information please visit [MolgenisArmadillo](https://molgenis.github.io/molgenis-r-armadillo/index.html). + +Below some basic operations to get you started using MolgenisArmadillo: + +=== "Prerequisite" + + ???+ example + + ```r linenums="1" title="Prerequisite.r" + # install the following packages + install.packages("MolgenisArmadillo") + install.packages("dplyr") + install.packages("arrow") + # After installation make sure you can load the libraries + library(MolgenisArmadillo) + library(dplyr) + library(arrow) + # verify the loaded libraries (see: other attached packages) + sessionInfo() + ``` +=== "Basic usage" + + ???+ example + + ```r linenums="1" title="MolgenisArmadillo.r" + ## Prerequisites + # + # - Run the code in Prerequisite.r + # + library(MolgenisArmadillo) + + ## + # To share your data using Armadillo, you first need to login + + ## Login + # + # In order to access the files as a data manager, you need to log in. + # The login method needs the URLs of the Armadillo server. + # It will open a browser window where you can identify yourself with the ID provider. + # + # If you are unsure how this login function (or any function) works ask R to provide documentation + ?armadillo.login + + armadillo.login( + armadillo = "https://armadillo.test.molgenis.org" + ) + #token <- armadillo.get_token("https://armadillo.test.molgenis.org") + + # armadillo.login will create a session and store the credentials in the environment. + + ## Structure + # To share data via Armadillo you can have a nested structure to save you data. + + # We distinguish: + # - projects + # - folders + # - tables + + ### Projects + # Projects are root-folders you can give persons permissions on. + # you can imagine that you will use a separate project for each study you need to support. + # This way you make sure people can not see each others variables. + + ### Folders + # Folder objects can be used to version the different tables you want to share in Armadillo. + # This is not mandatory and are free to use the folder level as you see fit. + # In our examples we will go into the versioning part a bit deeper. + + ### Tables + # Tables contain the data you want to share. + # This can be all the data on a certain subject, mostly used in consortia or a specific study you want to expose. + + ## Sharing data + # Assume you are in a consortia which has core-variables and outcome-variables. + # You want to share and version the whole dataset to all researchers which applied to access your data. + + # First we will create the project. In our case "ipec". + + cohort <- "workshop2" + armadillo.create_project(cohort) + + # Secondly we will load the table(s) we want to upload to Armadillo in the R-environment. + # We have test data which is in `arrow` format, the upload will take any object that has a table like structure to upload into the Armadillo. + # This can be SPSS, STATA, SAS or R-based data as well. + + library(arrow) + + # load the core data (make sure you are working in the correct directory) + nonrep <- arrow::read_parquet("data/nonrep.parquet") + yearlyrep <- arrow::read_parquet("data/yearlyrep.parquet") + + # explore your data (sanity check) + View(nonrep) + dim(nonrep) + names(nonrep) + table(nonrep$recruit_age) + + # The third step is determining the second level, which contains in this case the datamodel-version the type of variables and the data-version. + + # y_y-#variable-type#-x_x + + # y_y = datamodel version + # x_x = data version + + # upload the core variables + armadillo.upload_table(cohort, "2_1-core-1_0", nonrep) + armadillo.upload_table(cohort, "2_1-core-1_0", yearlyrep) + + ## Looking at the data + # There are helper functions to help you determine what is in the storage server. + # You can list projects and tables to what's in the storage. + + # list of projects + armadillo.list_projects() + + # listing tables per project + armadillo.list_tables(cohort) + + # You can download the data in the R-environment as well. + + # download table to local R environment + download_nonrep <- armadillo.load_table(cohort, "2_1-core-1_0", "nonrep") + + # check the column names from the local environment + colnames(download_nonrep) + + # check if local data and uploaded data are equal (optional) + setequal(nonrep, download_nonrep) + + # Now you can also take a look at the files in the user interface of the MinIO file server + # open the MinIO server URL in a browser window (used in armadillo.login). + + # > !IMPORTANT: run this part after subsetting the data + + ## Deleting the data + # To delete the data you need to throw away the contents first. + + # throw away the core tables + armadillo.delete_table(cohort, "2_1-core-1_0", "nonrep") + armadillo.delete_table(cohort, "2_1-core-1_0", "yearlyrep") + + # Now you can delete the project. + armadillo.delete_project(cohort) + ``` +=== "Create subsets" + + ???+ example + + ```r linenums="1" title="Creating_data_subsets_Armadillo.r" + ## Creating data subsets on the Armadillo server + # When researchers request access to your data they may in many cases not be granted access to the whole data set, but only to a subset. + # Here you can see the different relevant steps you need to take to create these subsets. + # + # - Setting up the environment + # - Logging into the servers + # - Exploring the data + # - Making subsets of the data + # - Upload data subsets + # + # - Delete data subsets + + ## Setting up the environment + # - Run the code in 0_Prerequisite.r + + # load required libraries + library(MolgenisArmadillo) + library(dplyr) + + ## Logging in to the servers + # In order to access the files you need to log in using the URLs of the Armadillo server. + # A browser window will be opened where you can identify yourself with the ID provider. + + armadillo.login( + armadillo = "https://armadillo.test.molgenis.org" + ) + # A session will be created and the credentials are stored in the environment. + + ## Explore the data + # Let's assume you are in a consortium which has core-variables and outcome-variables. + # You want to share a subset of the whole data set with certain researchers that applied for access to your data. + + # List projects on the Armadillo server. + armadillo.list_projects() + + # Next create a study, here called 'subset1'. + # NOTE: change the name of the subset for your own practice run + + study <- "study1" + armadillo.create_project(study) + + # List the tables in a project + + # You want to share data from the cohort you just uploaded (MolgenisArmadillo.r). + # Change the cohort_name to your own cohort name here: + cohort = "workshop1" + + # List the available tables within this project. + armadillo.list_tables(cohort) + + # Subset the core variables + # Download the relevant core tables to the local environment + + nonrep <- armadillo.load_table(cohort, "2_1-core-1_0", "nonrep") + yearlyrep <- armadillo.load_table(cohort, "2_1-core-1_0", "yearlyrep") + + # List their variables + + colnames(nonrep) + colnames(yearlyrep) + + # Subset the variables that were requested per table. + + subset_core_nonrep <- nonrep %>% select(child_id, asthma_m, preg_cig, preg_fever, preg_alc) + subset_core_yearlyrep <- yearlyrep %>% select(child_id, cohab_, smk_exp) + + ## Uploading the data subset + # Check the variables in the data subset before uploading + + colnames(subset_core_nonrep) + colnames(subset_core_yearlyrep) + + # Upload the data subsets + armadillo.upload_table(study, "2_1-core-1_0", subset_core_nonrep, "nonrep") + armadillo.upload_table(study, "2_1-core-1_0", subset_core_yearlyrep, "yearlyrep") + + # See if tables are uploaded + armadillo.list_tables(study) + + # Now you can also take a look at the files in the user interface of the Armadillo server. In this case: https://armadillo.test.molgenis.org/#/projects + + # > !IMPORTANT: run this part after subsetting the data + + ## Deleting the data + # To delete the data you need to throw away the contents first. + + # throw away the core tables + armadillo.delete_table(study, "2_1-core-1_0", "nonrep") + armadillo.delete_table(study, "2_1-core-1_0", "yearlyrep") + + # Now you can delete the project. + + armadillo.delete_project(study) + ``` + +!!! note + The example files used in the examples above can be found here: [nonrep.parquet](../../data/nonrep.parquet), [yearlyrep.parquet](../../data/yearlyrep.parquet) diff --git a/docs/pages/contact.md b/docs/pages/contact.md new file mode 100644 index 000000000..a0a7d9e40 --- /dev/null +++ b/docs/pages/contact.md @@ -0,0 +1,3 @@ +# Contact + +MOLGENIS Support (support@molgenis.org). \ No newline at end of file diff --git a/docs/pages/dev_guides.md b/docs/pages/dev_guides.md new file mode 100644 index 000000000..a12f56a59 --- /dev/null +++ b/docs/pages/dev_guides.md @@ -0,0 +1,176 @@ +# Developer Guidelines +The Armadillo and DataSHIELD community both are very welcoming to anyone who wants to contribute in any sort of form. +One way of doing that is by helping with the development of Armadillo. To help you get started, we've put together some +information to get you started and help you get familiarised with our code and way of working. + +=== ":fontawesome-solid-circle-info: General information" + +
Commits and automatic releases
+ + Versionnumbers are updated according to [semantic versioning](https://semver.org/), + using [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/). + + It is especially important to correctly identify and commit user-facing changes. These being: bugfixes and new + features. The correct prefix to a commit will trigger an automatic release. Automatic releases are set to be + pre-releases. + + Each commit with `!` just before the colon `:` is a major update, indicating a breaking change. So use it wisely. + You can also add `BREAKING CHANGE:` in the long commit message format. + + - Use `feat!: ...` or `fix!: ...` for a major upgrade, indicating a breaking change. + - Use `feat: ...` for a minor upgrade, indicating a new feature. + - Use `fix: ...` for a patch update, indicating a bugfix. + +Releases
+ As mentioned above, the automatic releases in Armadillo are pre-releases. Normal releases are done manually whenever + deemed necessary. Of course a new release can be requested anytime. + +Testing and continuous integration
+ All our repositories contain unittests. Although we usually aim for a coverage of 80%, sometimes we make an + exception. Tests are ran as part of our continuous integration, on each pull request and each merge with the master. + +Pull request
+ We encourage everyone to contribute to MOLGENIS armadillo. We do however have a couple of requirements for a pull + requests, mostly because of our automatic processes. + + 1. Start the title of your PR with prefix using conventional commits (see general information tab) + 2. When you're fixing an issue, include "closing #issuenumber" or "fixes #issuenumber" in a commit in the PR + 3. Describe what your PR does shortly and how to test it. + +Armadillo API and Swagger
+ Armadillo can be controlled using its REST API. To see and test all available endpoints, you can visit Armadillo's + swaggerpage. Swagger is a very useful tool for exactly this purpose. The page can be found on + `/swagger-ui/index.html` of each Armadillo instance. If you want to see an example of our swagger page, you can + visit it on [our demo server](https://armadillo-demo.molgenis.net/swagger-ui/index.html). + + { width="700" } + ///caption + Example of an endpoint tested our using our swagger page. + /// + +=== ":fontawesome-brands-java: Java" + + The serverside code of MOLGENIS Armadillo + ([molgenis-service-armadillo](https://github.com/molgenis/molgenis-service-armadillo)) is written in Java. In order + to run it locally you will need the following prerequisites: + + - Java 17 (unless running with Docker) + - Docker + +Project Structure
+ If you look at the java code in [our repository](https://github.com/molgenis/molgenis-service-armadillo/), you might + notice there are two parts to it: + + - [Armadillo](https://github.com/molgenis/molgenis-service-armadillo/tree/master/armadillo) + - [R](https://github.com/molgenis/molgenis-service-armadillo/tree/master/r) + + The Armadillo part is the main application that contains all basic funcitonality and APIs. It uses the R part to + communicate with R, using DataSHIELD logic. + +Tools
+pre-commit
+ This repository uses `pre-commit` to manage commit hooks. An installation guide can be found + [here](https://pre-commit.com/index.html#1-install-pre-commit). To install the hooks, run `pre-commit install` from the root folder of this repository. Now + your code will be automatically formatted whenever you commit. The pre-commit hooks we've set, will reformat new + code upon commit, so that it matches our code style. + +Running
+ There are several ways to run armadillo. What you're planning on doing with armadillo determines the best way to go. + If you just want to test how it works, runnig via the jar or even using docker is very suitable. If you're planning + on writing code yourself, it's better to clone the project, build it yourself and run it, either using your IDE (we + use IntelliJ), or running the jar you built locally. + +Jar
+ + 1. Download the jar from our + [releases page](https://github.com/molgenis/molgenis-service-armadillo/releases). + 2. Copy paste the contents of + [application-template.yml](https://github.com/molgenis/molgenis-service-armadillo/blob/master/application.template.yml) + and paste it in a file called `application.yml`, in the same folder as the downloaded jar. + 3. To start the application, run `java -jar molgenis-armadillo-x.yy.zz.jar`. + 4. Go to `http://localhost:8080` to see the Armadillo UI. + +Docker
+ For testing without having to installing Java you can run using docker: + + 1. Install [docker-compose](https://docs.docker.com/compose/install/) + 2. Download this [docker-compose.yml](docker-compose.yml). + 3. Execute `docker-compose up` + 4. Once it says 'Started', go to http://localhost:8080 to see your Armadillo running. + + The command must run in the same directory as the downloaded docker file. + We made docker available via 'docker.sock' so we can start/stop DataSHIELD profiles. + Alternatively you must include the datashield profiles into this docker-compose. + You can override all application.yml settings via environment variables (see commented code in docker-compose file). + +IntelliJ
+ We develop Armadillo using IntelliJ. To do so: + + 1. Clone our project: + ```shell + git clone https://github.com/molgenis/molgenis-service-armadillo.git + ``` + 2. Open the project in Intellij. + 3. Make sure the right version of Java is set in `File> Project structure...` + 4. Build the project: select the gradle button (elephant symbol, usually on the right of your intellij screen), + select `clean` and then `build`. + 5. Go to the `ArmadilloServiceApplication` class (press shift-shift and search for it). + 6. Click on the play button in front of the main function. You might have to press "Run" again if a popup with the + run configuration appears. + +Testing
+ Most of our code is covered by unittests. Although we usually aim for a coverage of 80%, sometimes we make + exceptions because we value quality of tests over their quantity. We recognise that for some pieces of code it's + hard to write meaningful unittests (half of some functions would have to be mocked, usually causing tests to + lose their purpose). This is especially the case in our Java repository. Of course we don't want + these pieces untested, therefore we have + [a set of release test scripts](https://github.com/molgenis/molgenis-service-armadillo/tree/master/scripts/release). + This test is made to test basic functionality from Datamanager and Researcher perspective using the most commonly + used DataSHIELD packages. This test script is used before each release to ensure Armadillo's quality. The script + also runs partly (without OIDC) in our continuous integration test on each pull request, as well as the unittests. + +host.docker.internal error
+ This error is returned only in development environments, when working on an unsupported operating system and running + a profile that has the `resourcer` R package whitelisted. The only way to fix this error, is by temporarily altering + the Armadillo source code in the + [DockerService.java](https://github.com/molgenis/molgenis-service-armadillo/blob/master/armadillo/src/main/java/org/molgenis/armadillo/profile/DockerService.java) + Go to the `install_image` method to where `createContainerCmd` is called: + ```java + cmd.withExposedPorts(exposed) + ... + .exec() + ``` + Simply add `.withExtraHosts("host.docker.internal:host-gateway"))` before the `.exec()`. To ensure the code is + properly updated, we suggest rebuilding the code before restarting Armadillo. If you already had the profile + running, you will need to restart that as well. +=== ":material-vuejs: JavaScript/VueJS" + + The user interface (UI) of MOLGENIS Armadillo is written in JavaScript, using VueJS as framework. We use `yarn` to + compile the code and develop in VSCode. Our setup is as follows: + + 1. Open the armadillo repository, and build and run it in InteliJ (as described in Java Developer guide) + 2. Open the ui folder of the repository in VSCode. + 3. Install using: + ```shell + yarn install + ``` + 4. Run dev server using: + ``` + yarn dev + ``` + You can now login on http://localhost:8080 and then switch to http://localhost:8081 to directly see the changes + you're changing the UI code. + +=== ":simple-r: R" + + We maintain several R packages: + + - [MolgenisArmadillo](https://github.com/molgenis/molgenis-r-armadillo) + - [DSMolgenisArmadillo](https://github.com/molgenis/molgenis-r-datashield) + - [DSUpload](https://github.com/lifecycle-project/ds-upload) + - [DSTidyVerse](- [DSTidyverse](https://github.com/molgenis/ds-tidyverse)) + - [DSTidyverseClient](https://github.com/molgenis/ds-tidyverse-client) + + We aim to release all our packages (excluding DSUpload) to CRAN, to increase visibility and compatibility. This + means that documentation and vignettes should be updated in every pull request that changes or updates + functionality. \ No newline at end of file diff --git a/docs/pages/examples_usage.md b/docs/pages/examples_usage.md new file mode 100644 index 000000000..d297c8047 --- /dev/null +++ b/docs/pages/examples_usage.md @@ -0,0 +1,6 @@ +# Example of usage +User management + +Project management + +Profile management \ No newline at end of file diff --git a/docs/pages/faq.md b/docs/pages/faq.md new file mode 100644 index 000000000..52ecc6d61 --- /dev/null +++ b/docs/pages/faq.md @@ -0,0 +1,95 @@ +# Frequently Asked Questions + +??? question "Docker gives a `java.socket` error" + You might need to enable Docker socket. On Docker desktop you can find that under 'settings' and 'advanced'. + +??? question "Can I use docker compose to start profiles?" + Instead of making Armadillo start/stop DataSHIELD profiles you can also use docker compose. See commented section in docker-compose.yml file. + +??? question "Can I pass environment or commandline variables instead of application.yml?" + Yes, it is standard spring. + +??? question "Can I run Armadillo with oauth2 config offline?" + Yes, you can run in 'offline' profile + ```bash + ./gradlew run -Dspring.profiles.active=offline + ``` + +??? question "How to run Armadillo version 2?" + For armadillo 2.x you can follow [instructions](https://github.com/molgenis/molgenis-service-armadillo/tree/armadillo-service-2.2.3) for testing we use docker compose. For production environment we are using [Ansible](https://galaxy.ansible.com/molgenis/armadillo) + +??? question "How to run Armadillo as developer?" + We develop Armadillo using IntelliJ. + +??? question "How to build Armadillo" + Run following command in the github root: + ```./gradlew build``` + + To execute in 'dev' run following command in the github root: + ```./gradlew run``` + + **Setting up development tools** + + This repository uses `pre-commit` to manage commit hooks. An installation guide can be found [here](https://pre-commit.com/index.html#1-install-pre-commit). To install the hooks, run `pre-commit install` from the root folder of this repository. Now your code will be automatically formatted whenever you commit. + + **How to change data directory** + + Data is automatically stored in the `data` folder in this repository. You can choose another location in `application.yml` by changing the `storage.root-dir` setting. + + When you run Armadillo locally for the first time, the `lifecycle` project has not been added to the system metadata yet. To add it automatically, see Application properties. + Alternatively you can add it manually: + + - Go to the Swagger UI (`http://localhost:8080/swagger-ui/index.html`) + - Go to the `PUT /access/projects` endpoint + - Add the project `lifecycle` + + **Working with resources in development mode** + + When developing locally, docker has trouble connecting to localhost. This problem becomes clear when working with + resources. Luckily there is a quick fix for the problem. Instead of defining a resource as for example + `http://localhost:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda`, rewrite it to: + `http://host.docker.internal:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda`. Here's some example R code + for uploading resources: + + ```R + ## Uploading resources + library(MolgenisArmadillo) + library(resourcer) + + token <- armadillo.get_token("http://localhost:8080/") + + resGSE1 <- resourcer::newResource( + name = "GSE66351_1", + secret = token, + url = "http://host.docker.internal:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda", + format = "ExpressionSet" + ) + + armadillo.login("http://localhost:8080/") + armadillo.upload_resource(project="omics", folder="ewas", resource = resGSE1, name = "GSE66351_1") + ``` + And for using them: + ```R + library(DSMolgenisArmadillo) + library(dsBaseClient) + + token <- armadillo.get_token("http://localhost:8080/") + + builder <- DSI::newDSLoginBuilder() + builder$append( + server = "local", + url = "http://localhost:8080/", + token = token, + driver = "ArmadilloDriver", + profile = "uniform", + resource = "omics/ewas/GSE66351_1" + ) + + login_data <- builder$build() + conns <- DSI::datashield.login(logins = login_data, assign = TRUE) + + datashield.resources(conns = conns) + datashield.assign.resource(conns, resource="omics/ewas/GSE66351_1", symbol="eSet_0y_EUR") + ds.class('eSet_0y_EUR', datasources = conns) + datashield.assign.expr(conns, symbol = "methy_0y_EUR",expr = quote(as.resource.object(eSet_0y_EUR))) + ``` diff --git a/docs/ops/installing.md b/docs/pages/install_management/armadillo_install.md similarity index 64% rename from docs/ops/installing.md rename to docs/pages/install_management/armadillo_install.md index 3e649b567..d999ae7af 100644 --- a/docs/ops/installing.md +++ b/docs/pages/install_management/armadillo_install.md @@ -1,6 +1,7 @@ # Armadillo installation -> This guide assumes using Ubuntu Server LTS as the installation script expect the `systemd` service. See [Install alternatives](#install-alternatives) below. +!!! note + This guide assumes you are using Ubuntu Server LTS, the installation script expects the `systemd` service. See [Install alternatives](#install-alternatives) below. ## Requirements @@ -14,18 +15,18 @@ You need a server or virtual machine to deploy the Armadillo stack. The specific | 20.000-70.000 | 16 | 100 | 4 | | 70.000 > | 32 | 150 | 8 | -In case of using dsOmics this setup can be rather bigger. Please contact molgenis-operations@umcg.nl for the latest specifications. +In case of using dsOmics this setup can be rather bigger. Please [contact](../contact.md) us for the latest specifications. ## Software requirements -* Java 17 JRE or JDK +* Java 19 JRE or JDK * Docker -In addition to these, there are other optional components you may wish to install, such as setting up nginx as a reverse proxy. +In addition to these, there are other optional components you may wish to install, such as setting up nginx as a reverse proxy. ## Domain -An domain or an hostname ie `cohort.armadillo.domain.org` is required to run Armadillo. +A domain or an hostname ie `cohort.armadillo.domain.org` is required to run Armadillo. This domain is needed for the installation script. @@ -33,15 +34,15 @@ This domain is needed for the installation script. Before we start with the deployment of Armadillo you will need to register your domain that you are going to use with your Armadillo on the DataSHIELD authentication server. This allows you to delegate the authentication and user management. The authorization will still be under the control of the Data Manager (who gets access and who don't get access) within your armadillo installation. -To registrate you will need to send a mail to `molgenis-support@umcg.nl` with the [chosen domains](#domain) and the e-mail adres of the Data Manager who is granted admin permissions in Armadillo. Also add to the mail that you want to register for the the DataSHIELD authentication server and if you belong to a project like Lifecycle, Athlete or Longitools. +To register you will need to [contact](../contact.md) us with the [chosen domains](#domain) and the e-mail address of the Data Manager who is granted admin permissions in Armadillo. Also add to the mail that you want to register for the the DataSHIELD authentication server and if you belong to a project like Lifecycle, Athlete or Longitools. -When the Armadillo domain is registrerd you will get an mail back with data that need to be inserted in step 2. +When the Armadillo domain is processed you will get an email back with data that need to be inserted in step 2. The values needed are: -- OIDC service url i.e. https://lifecycle-auth.molgenis.org -- OIDC Client ID -- OIDC Client Secret + - OIDC service url i.e. https://lifecycle-auth.molgenis.org + - OIDC Client ID + - OIDC Client Secret ## Securing the connection @@ -55,7 +56,7 @@ setsebool -P httpd_can_network_connect on ## Installing Armadillo as service -We run Armadillo in production as a Linux service on Ubuntu, ensuring it gets restarted when the server is rebooted. You might be able to reproduce also on CentOS (using yum instead of apt). +We run Armadillo in production as a Linux service on Ubuntu, ensuring it gets restarted when the server is rebooted. You might be able to reproduce also on CentOS (using `yum` instead of `apt`). ### 1. Install necessary software @@ -65,17 +66,18 @@ apt install openjdk-19-jre-headless apt install docker.io ``` -Note: you might need 'sudo' +??? note + you might need to execute these commands as `sudo` ### 2. Run installation script -This step will install most recent [release](https://github.com/molgenis/molgenis-service-armadillo/releases) +This step will install the latest [release](https://github.com/molgenis/molgenis-service-armadillo/releases). After installation the Armadillo application is installed with the given configuration and its service. #### 2.1 Download the setup script -``` +```bash wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/master/scripts/install/armadillo-setup.sh ``` @@ -95,18 +97,17 @@ The installation script requires some arguments: | admin-password | Secure password for the admin user | | datadir | The location where the data is stored. This directory should be have enough diskspace en could be backuped (Default → /usr/share/armadillo/data)| | domain | The URL where armadillo is listening on. For example: cohort.armadillo.domain.org | -||| -|oidc | Enable OIDC, see [authentication](#Authentication) | +|oidc | Enable OIDC, see [authentication](#authentication) | |oidc_url | Given oidc URL | |oidc_clientid | Given client ID| |oidc_clientsecret | Given secret ID| -### 2.2 Run the install script +#### 2.2 Run the install script Adapt the following install command to suit your situation. Use `--help` to see all options. -> Note: https://lifecycle-auth.molgenis.org is MOLGENIS provided OIDC service but -you can also use your own, see FAQ below. +??? note + https://lifecycle-auth.molgenis.org is MOLGENIS provider of the OIDC service but you can also use your own. ```bash bash armadillo-setup.sh \ @@ -119,7 +120,7 @@ bash armadillo-setup.sh \ --oidc_clientsecret secret \ ``` -## File locations +### File locations The script creates using default values the follow files and directories: @@ -142,12 +143,15 @@ systemctl start armadillo After the installation is complete Armadillo is listening on port 8080. Test the setup by visiting `http://armadillo.cohort.study.com:8080` or type in the terminal `wget http://localhost:8080` to see a text response. -> Note: the Armadillo website is not secure yet so you need to setup a *front-end* proxy. +??? note + The Armadillo website is not secure yet so you need to setup a *front-end* proxy. -When having setup this *front-end* proxy you could get a bad gateway. This mostly means Armadillo is not ready yet as you already have tested armadillo as mentioned above. +When having setup this *front-end* proxy you could get a bad gateway (see below). This mostly means Armadillo is not ready yet as you already have tested armadillo as mentioned above. -> 502 Bad Gateway -> nginx/1.18.0 (Ubuntu) +```http +502 Bad Gateway +nginx/1.18.0 (Ubuntu) +``` Recheck with `wget http://localhost:8080` and check the log files. @@ -171,23 +175,55 @@ server { } ``` -> Note: the above is still not secure but not you can reach Armadillo from `http://armadillo.cohort.study.com`. +??? note + The above is still not secure but not you can reach Armadillo from `http://armadillo.cohort.study.com`. To secure the communication using https we have a [nginx example](https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/master/scripts/install/conf/armadillo-nginx.conf) ## Backups -A good start for backuping data is the /usr/share/armadillo and /etc/armadillo. If you gave another datadir as setup option you also should backup this directory. For disaster backups you should contact your IT department. +A good start for data backup is the `/usr/share/armadillo` and `/etc/armadillo`. If you gave another datadir as setup option you also should backup this directory. For disaster backups you should contact your IT department. ## Install alternatives -- On local machine using [java](./install/install_java.md) -- Armidillo running as a [Docker](./install/install_docker.md) container. -- [Apache](./install/install_apache.md) - -For questions on other linux release you can email molgenis-operations@umcg.nl +- On local machine using java +- Armidillo running as a Docker container. +- [Apache](#apache) ## What's next? * [For the server owner or data manager who need to put data on to the server](https://molgenis.github.io/molgenis-r-armadillo/) * [For the researcher who want to start analyzing the data on the server](https://molgenis.github.io/molgenis-r-datashield/) + +## Apache + +It is possible to run Molgenis Armadillo using Apache, however we do **not** provide support with this configuration. + +### Encoding + +Apache requires some additional configuration to get the `/storage/projects/{project}/objects/{object}` to work. When this endpoint doesn't work: + + - tables cannot be assigned + - subsets cannot be created + - resources cannot be used. + +### Tell Armadillo about https + +We need to tell Armadillo server how to building URLs. + +### Changes to your site-enabled configuration + +Your configuration probably should look like this. + +```conf +ProxyPreserveHost On + +ProxyPass / http://localhost:8080/ nocanon + +AllowEncodedSlashes On + +RequestHeader set X-Forwarded-Proto https +RequestHeader set X-Forwarded-Port 443 +``` + +After setting this don't forget to restart Apache. \ No newline at end of file diff --git a/docs/pages/install_management/armadillo_management.md b/docs/pages/install_management/armadillo_management.md new file mode 100644 index 000000000..1137a59f2 --- /dev/null +++ b/docs/pages/install_management/armadillo_management.md @@ -0,0 +1,19 @@ +# Armadillo management + +Armadillo has three main screens to manage projects, user access and DataSHIELD profiles: + +## Create data access projects + +Data managers can use the Armadillo web user interface or [MolgenisArmadillo R client](https://molgenis.github.io/molgenis-r-armadillo) to create 'projects' and upload their data into those. Data tables need to be in parquet format that supports fast selections of the columns (variables) you need for analysis. Other files can be configured as 'resources'. + +## Manage user access + +Data managers can use the permission screen to give email addresses access to the data. Everybody signs in via single sign on using an OIDC central authentication server such as KeyCloack or Fusion auth that federates to authentication systems of connected institutions, ideally using a federated AAI such as LifeScience AAI. + +## Configure DataSHIELD profiles + +To analyse data, users must choose a datashield profile. Armadillo owners can use the web user interface to configure new profiles. Assuming you installed docker you can also start/stop these images. Alternatively you can use docker-compose for that. We recommend selecting one of the DataSHIELD standard profiles. + +## End users can use Armadillo as any other DataSHIELD server + +A researcher connects from an [R client](https://molgenis.github.io/molgenis-r-datashield) to one or multiple Armadillo servers. The data is loaded into an R session on the Armadillo server specifically created for the researcher. Analysis requests are sent to the R session on each Armadillo server. There the analysis is performed and aggregated results are sent back to the client. \ No newline at end of file diff --git a/docs/upgrade-3-4.md b/docs/pages/install_management/armadillo_migrate_2_to_3.md similarity index 88% rename from docs/upgrade-3-4.md rename to docs/pages/install_management/armadillo_migrate_2_to_3.md index c4f477029..a6cf5b8e0 100644 --- a/docs/upgrade-3-4.md +++ b/docs/pages/install_management/armadillo_migrate_2_to_3.md @@ -1,6 +1,9 @@ -# Upgrading to Armadillo 4 (rock only) +# Migrate Armadillo 3 to Armadillo 4 -> Note: We assume Ubuntu with systemd is used. +Upgrade to Armadillo 4 (rock only) + +??? note + We assume Ubuntu with systemd is used. The upgrade from Armadillo v3.4 to 4.x is breaking as the profiles must be Rock profiles. @@ -16,10 +19,10 @@ Make a note of the version as you will use this below. Check if the new profiles are compatible with your needs, these profile names can be edited later on in the manual: -- `datashield/rock-base:latest` -- `datashield/rock-dolomite-xenon:latest` + - datashield/rock-base:latest + - datashield/rock-dolomite-xenon:latest -See also DataSHIELD [profiles](https://www.datashield.org/help/standard-profiles-and-plaforms) +See also DataSHIELD profiles ## 2. Check server space @@ -31,7 +34,8 @@ Make sure enough disk space is available for the Rock only images. # Check disk space df -H ``` -If you have 15 GB or more available, you can continue. If you have less available, check `docker image list` to see if you can cleanup some docker images (you only need the latest `datashield/armadillo-rserver` and `datashield/armadillo-rserver_caravan-xenon` for armadillo 3). + +If you have 15 GB or more available, you can continue. If you have less available, check `docker image list` to see if you can cleanup some docker images (you only need the latest `datashield/armadillo-rserver` and `datashield/armadillo-rserver_caravan-xenon` for armadillo 3). ### 2.2 Check docker images @@ -70,7 +74,7 @@ df -H ## 3. Download required files -Make a note of the version number ie. `v4.1.3` as you need to download some files from the terminal using the updatescript. +Make a note of the version number ie. `v4.1.3` as you need to download some files from the terminal using the update script. ### 3.1 Update script @@ -100,7 +104,8 @@ chmod u+x armadillo-check-update.sh You can run the following script to download the new Armadillo version. -> The output could help us to help you fix problems. +??? note + The output could help us to help you fix problems. ```bash # Change the version number v4.x.y @@ -117,17 +122,18 @@ ls -ltr /usr/share/armadillo/application/ ## 4. Config the new version ### 4.1 application.yml -NOTE: To compare the latest template to your own configuration, see the troubleshooting section below. The safest way to update armadillo is by fetching the template and filling it in with your configuration using the information in the troubleshooting section. You can try the following first: + +To compare the latest template to your own configuration, see the troubleshooting section below. The safest way to update armadillo is by fetching the template and filling it in with your configuration using the information in the troubleshooting section. You can try the following first: Edit the application.yml: + ```bash nano /etc/armadillo/application.yml ``` Below the line `docker-management-enabled: true`, ensure to insert the line `docker-run-in-container: false`. Typically, you'll find these configurations at the beginning of the file. - -### 4.2 Make backup of system config +## 4.2 Make backup of system config ```bash # Still in the correct directory? (`/root/v4.x.y`) @@ -182,29 +188,27 @@ systemctl status armadillo ## 6. Log on to the UI -Go to your armadillo website. Is the version in the left top corner updated? This means the update was successful. We're -almost finished. +Go to your armadillo website. Is the version in the left top corner updated? This means the update was successful. ## 7. Update profiles + Login into the website and go to the profiles tab. Here two profiles should be listed: `default` and `xenon`. Any other profiles can be removed. -1. Edit the default profile. -2. Change the "image" to `datashield/rock-base:latest` and save. +1. Edit the default profile. +2. Change the "image" to `datashield/rock-base:latest` and save. 3. Start the default profile. 4. Edit the "xenon" profile. 5. Change the "image" to `datashield/rock-dolomite-xenon:latest` and save. -6. Start the xenon profile. +6. Start the xenon profile. Everything should now be working correctly. You can try and login to your server via the central analysis server, using -the `DSMolgenisArmadillo` (2.0.5 or up) package to test. +the `DSMolgenisArmadillo` (2.0.5 or up) package to test. -Enjoy =) -Team Armadillo +## Troubleshooting +### Logs -## Troubleshooting -#### Logs Reviewing the log files can provide valuable insights into any issues or activities within the application. If you encounter any errors or unexpected behavior, examining the log files can often help diagnose the problem. Check log files location @@ -242,8 +246,8 @@ or tail -f /var/log/armadillo/* ``` +### Compare application.yml -#### Compare application.yml Although we try to be very complete in this manual, if you run into issues, it might be because a setting was changed in the application.yml. You can check if application settings has any new entries by first downloading the application template (for reference). @@ -267,7 +271,7 @@ Your output should look like output below. - the `>` means we have a setting (probably added or options) - the `|` means both have different values which happens with OICD/oauth settings for sure. -``` +```bash armadillo: armadillo: # set this false if you DON'T want Armadillo to create/edit # set this false if you DON'T want Armadillo to create/edit docker-management-enabled: true docker-management-enabled: true diff --git a/docs/upgrade-2-3.md b/docs/pages/install_management/armadillo_migrate_3_to_4.md similarity index 70% rename from docs/upgrade-2-3.md rename to docs/pages/install_management/armadillo_migrate_3_to_4.md index 0b7042502..aabef5e3d 100644 --- a/docs/upgrade-2-3.md +++ b/docs/pages/install_management/armadillo_migrate_3_to_4.md @@ -1,19 +1,18 @@ -### Migrate Armadillo 2 to Armadillo 3 +# Migrate Armadillo 2 to Armadillo 3 Migrating from Armadillo 2 to Armadillo 3 can be done in two variants, a full migration including [projects, users and data](#migrate-projects-users-and-data) or [just projects and their users](#migrate-projects-and-their-users). Both options require Python (version 3.8) and additional python libraries, described in [Getting started](#getting-started). -### Getting started +## Getting started To start the migration, python 3.8 is advised together with a number of utilitarian python libraries. Other python versions might work, but performance has only been tested with python 3.8. -#### Install with Python virtual environment +### Install with Python virtual environment For more info see [python virtual environment](https://docs.python.org/3/library/venv.html). -The following code does not require superuser rights. The code does assume you are already -in the [scripts/upgrade](https://github.com/molgenis/molgenis-service-armadillo/tree/master/scripts/upgrade) directory. +The following code does not require superuser rights. The code does assume you are already in the [scripts/upgrade](https://github.com/molgenis/molgenis-service-armadillo/tree/master/scripts/upgrade) directory. ```bash python3 -m venv venv @@ -21,34 +20,44 @@ source ./venv/bin/activate pip install -r requirements.txt ``` -If the installation of one (or more libraries) fails, try to install the libraries one by one. +??? tip + If the installation of one (or more libraries) fails, try to install the libraries one by one. -### Migrate Projects, users and data +## Migrate Projects, users and data -#### 1. Check if there's enough space left on the server -``` +### 1. Check if there's enough space left on the server + +```bash df -h ``` + Compare to: -``` + +```bash du -h /var/lib/minio ``` -Available space should be at least twice the size of the MinIO folder. -#### 2. Backup Armadillo 2 settings -``` +??? warning + Available space should be at least twice the size of the MinIO folder. + +### 2. Backup Armadillo 2 settings + +```bash mkdir armadillo2-backup rsync -avr /usr/share/armadillo armadillo2-backup cp /etc/armadillo/application.yml armadillo2-backup/application-armadillo2.yml ``` -N.B.change /usr/share to path matching your local config. -#### 3. Install helper software +???+ note + Change `/usr/share` to the path matching your local config. + +### 3. Install helper software FIXME: Skip these steps as you have already done installing the python code in a virtual env. Login to your server as root, using ssh. -``` + +```bash apt update #apt install pip @@ -56,36 +65,51 @@ apt update #pip install fusionauth-client #pip install simple_term_menu ``` + If you get a purple message asking to update, accept and install everything. + Restart of server is recommended after this. -N.B. Note that the commands in this manual are for Ubuntu, on other linux systems, -the `apt` command needs to be replaced with the correct one. +??? note + The commands in this manual are for Ubuntu, on other linux systems, the `apt` command needs to be replaced with the correct one. + +### 4. Stop all docker images for Armadillo 2 -#### 4. Stop all docker images for Armadillo 2 List all docker images -```docker ps -a``` -Stop and remove all Armadillo 2 related images (except for MinIO), e.g. +```bash +docker ps -a ``` + +Stop and remove all Armadillo 2 related images (except for MinIO), e.g. + +```bash docker rm armadillo_auth_1 armadillo_console_1 armadillo_rserver-default_1 armadillo_rserver-mediation_1 armadillo_rserver-exposome_1 armadillo_rserver-omics_1 armadillo_armadillo_1 -f ``` -Check with `docker ps -a` if there are still containers running, if so remove these (except for the MinIO) in the same way as the others. -#### 5. Install armadillo -``` +Check with `docker ps -a` if there are still containers running, if so remove these (**except for the MinIO**) in the same way as the others. + +???+ warning + Make sure you do **not** remove the docker instance of MinIO before you migrated the data! + +### 5. Install armadillo + +```bash apt update apt install openjdk-19-jre-headless apt install docker.io ``` + The docker.io step might fail because containerd already exists, if that's the case, remove containerd and try again: -``` + +```bash apt remove containerd.io apt install docker.io ``` Get armadillo: -``` + +```bash wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/master/scripts/install/armadillo-setup.sh bash armadillo-setup.sh \ --admin-user admin \ @@ -97,62 +121,77 @@ bash armadillo-setup.sh \ --oidc_clientsecret secret \ --cleanup \ ``` + Don't forget to set a proper admin password (use a generator), domain, clientid and clientsecret. The client id and secret can be found on the lifecycle auth server in the configuration for your server. If you don't have permissions to receive this, you can ask the support team to get it for you. Open armadillo in the browser and try to login using basicauth to check if the server is running properly. If it is not running at all, try: -``` + +```bash systemctl start armadillo ``` -#### 6. Export data from Armadillo 2 into armadillo 3 -Look up the user/password in the application.yml of the old armadillo. They are called MinIO access key and minio -secret key. -``` +### 6. Export data from Armadillo 2 into armadillo 3 + +Look up the user/password in the application.yml of the old armadillo. They are called MinIO access key and minio secret key. + +```bash cat /root/armadillo2-backup/application-armadillo2.yml ``` + Do the following step in a separate screen. On ubuntu use: -``` + +```bash screen ``` + Navigate to the armadillo folder: -``` + +```bash cd /usr/share/armadillo ``` + This step will copy Armadillo 2 data from minio into the folder matching of an Armadillo 3 data folder: -``` + +```bash mkdir data wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/master/scripts/upgrade/migrate-minio.py python3 migrate-minio.py --minio http://localhost:9000 --target /usr/share/armadillo/data ``` -This might take a couple of minutes. You can detach the screen using `ctrl+a` followed by `d` and reattach it using -`screen -r`. +This might take a couple of minutes. You can detach the screen using ++ctrl+a++ followed by ++d++ and reattach it using `screen -r`. + +### 7. Run Armadillo 3 using exported data -#### 7. Run Armadillo 3 using exported data Make sure to move the exported data into the new 'data' folder. Optionally you might need to fix user permissions, e.g.: -``` + +```bash chown armadillo:armadillo -R data ``` + Check if armadillo is running by going to the URL of your server in the browser, login and navigate to the projects tab. -#### 8. Optionally, acquire a permission set from MOLGENIS team -If you previously ran a central authorisation server with MOLGENIS team, they can provide you with procedure to load -pre-existing permissions. They will use: -``` +### 8. Optionally, acquire a permission set from MOLGENIS team + +If you previously ran a central authorisation server with MOLGENIS team, they can provide you with procedure to load pre-existing permissions. They will use: + +```bash wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/master/scripts/upgrade/migrate-auth.py python3 migrate-auth.py --fusion-auth https://lifecycle-auth.molgenis.org --armadillo https://thearmadillourl.net ``` -Now check if all users and data are properly migrated. -NOTE: if the script fails with a timeout, try pinging the armadillo url and lifecycle auth url to see if they're reachable from the server. In case they are not, you could choose to export the users using the `export-users.py` script locally and then manually enter them into the system. +Now check if all users and data are properly migrated. + +??? note + If the script fails with a timeout, try pinging the armadillo url and lifecycle auth url to see if they're reachable from the server. In case they are not, you could choose to export the users using the `export-users.py` script locally and then manually enter them into the system. -#### 9. Cleanup ngnix config +### 9. Cleanup ngnix config Change `/etc/nginx/sites-available/armadillo.conf` to: -``` + +```bash server { listen 80; server_name urlofyourserver.org @@ -167,27 +206,33 @@ server { } } ``` -Note that the `https://` is missing in the server_name part. -NOTE: if port 443 and the SSL certificates are in the old config, you mind have to keep that part, so you should not comment that out. Keep the listen and certificate lines, comment out the rest and paste the config above below the existing config. -Remove the console, auth and storage file from: `/etc/nginx/sites-enabled/` and `/etc/nginx/sites-available/. +??? note + Note that the `https://` is missing in the server_name part. +??? note + If port 443 and the SSL certificates are in the old config, you mind have to keep that part, so you should not comment that out. Keep the listen and certificate lines, comment out the rest and paste the config above below the existing config. -``` +Remove the console, auth and storage file from: `/etc/nginx/sites-enabled/` and `/etc/nginx/sites-available/`. + +```bash systemctl restart nginx ``` -#### 10. Fix application.yml +### 10. Fix application.yml + Make sure the following is added: -``` + +```bash server: forward-headers-strategy: framework ``` -#### 11. Fix URLs in the lifecycle FusionAuth -Add the following to the config of your server: -`https://yourserver.com/login/oauth2/code/molgenis` +### 11. Fix URLs in the lifecycle FusionAuth + +Add the following to the config of your server: `https://yourserver.com/login/oauth2/code/molgenis` + +### 12. Set up profiles -#### 12. Set up profiles Login to armadillo in the browser. Navigate to the "Profiles" tab. Add a new profile with the following properties: Name: `xenon` @@ -196,21 +241,31 @@ Package whitelist: `dsBase`, `resourcer`, `dsMediation`, `dsMTLBase`, `dsSurviva Assign a random 9-number seed and create and start the container. -#### 13. Remove old MinIO data +### 13. Remove old MinIO data + First remove the MinIO docker container. First check the name of the container using `docker ps -a`, then: -``` + +```bash docker rm containername -f ``` + After that remove the data: -```rm -Rf /var/lib/minio/ ``` -### Migrate Projects and their users +```bash +rm -Rf /var/lib/minio/ +``` -Migration of just the projects and their users (with their corresponding rights) can be done -by using [export-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/export-users.py) and [import-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/import-users.py). -**This options does not migrate the data!** +???+ warning + Be sure you have migrated your data successfully or created a backup prior to deleting your minio data folder! -#### 1. Export Projects and users from Armadillo 2 +## Migrate Projects and their users + +Migration of just the projects and their users (with their corresponding rights) can be done by using [export-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/export-users.py) and [import-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/import-users.py). + +???+ warning + This options does not migrate the data! + +### 1. Export Projects and users from Armadillo 2 To export users from an Armadillo 2 server, one must use the [export-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/export-users.py) script. `export-users.py` can be used by using the following arguments: @@ -224,12 +279,13 @@ Empty projects (without users) will also be exported as an empty TSV (containing Also note that some projects might change in name, as Armadillo 3 is stricter with naming projects. Example: + ```bash pipenv shell python3 export-users.py -f https://armadillo2-server.org -o ./armadillo_2_exports ``` -#### 2. Import Projects and users TSVs into Armadillo 3 +### 2. Import Projects and users TSVs into Armadillo 3 To import users into an Armadillo 3 server, one must use the [import-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/upgrade/import-users.py) script. `import-users` can be used by using the following arguments: @@ -241,6 +297,7 @@ To import users into an Armadillo 3 server, one must use the [import-users.py](h Empty TSVs from [step 1](#1-export-projects-and-users-from-armadillo-2) will be imported as empty projects with no users. Example: + ```bash pipenv shell python3 import-users.py -s https://armadillo3-server.org -d ./armadillo_2_exports/2023-11-09 diff --git a/docs/minor-release-update.md b/docs/pages/install_management/armadillo_minor_release_update.md similarity index 66% rename from docs/minor-release-update.md rename to docs/pages/install_management/armadillo_minor_release_update.md index c04420b0f..f1324af94 100644 --- a/docs/minor-release-update.md +++ b/docs/pages/install_management/armadillo_minor_release_update.md @@ -1,12 +1,15 @@ -# Minor Version Upgrade Manual: Procedures for y.z Releases +# Armadillo minor release update -> Note: This manual is intended for minor version updates within the latest major release. For example, you can use it to update from version 4.1 to 4.7.1. For upgrading to a new major version, please refer to the specific manuals dedicated to major version upgrades. +## Minor Version Upgrade Manual: Procedures for y.z Releases -## Check latest version +???+ note + This manual is intended for minor version updates within the latest major release. For example, you can use it to update from version 4.1 to 4.7.1. For upgrading to a new major version, please refer to the specific manuals dedicated to major version upgrades. -For the latest 4.y.z release check https://github.com/molgenis/molgenis-service-armadillo/releases/latest. This will redirect to a v4.y.z page. +### Check latest version -# Updating Armadillo +For the latest 4.y.z release check [https://github.com/molgenis/molgenis-service-armadillo/releases/latest](https://github.com/molgenis/molgenis-service-armadillo/releases/latest). This will redirect to a v4.y.z page. + +## Updating Armadillo ### 1. Stop docker containers @@ -24,11 +27,11 @@ docker container list docker container stop
.
-
-The row will be opened in edit mode:
-
-
-
-The edit mode can be recognized by its blue background color and you have the option to add new users to your project
-by clicking on the + button 
of the users column. Then, you can either
-select an existing user from the dropdown, or add the email address of a new user.
-
-
-
-In case of adding a user in this screen, a warning will be shown to prevent email addresses with typographical errors from being added to
-your system. For example, if you have the address `j.doe@example.com` in your users table but attempt to add `j.die@example.com` a warning message will be displayed asking if you want to add a new user.
-
-
-
-It is not possible to edit the name of your project; this is intentional in order to ensure tables, resources,
-users, and permissions are transferred to the new project name correctly.
-
-Click on the checkmark 
to save the edited row and the X
-
to cancel. Be careful, if you do cancel your changes will be lost.
-
-### Adding projects
-
-To add a new project, click on the + button 
on top of the table. If
-you click this button, an empty row will be opened in edit mode.
-
-
-
-Click on the checkmark to save the edited row and the X
- to cancel. Be careful, if you do cancel your changes will be lost.
-
-## Project explorer
-
-If you click on the icon 
next to the project name, you will be directed to
-the `project editor`. In this screen you can upload and preview data in projects. To go back to your `projects` page,
-press the back button 
.
-
-
-
-If you click on a folder, it will open.
-
-
-
-Here you can upload files to that folder, or click on the tables (files) to preview their contents.
-
-
-
-To upload files, either drag a file from your file browser to the file upload area, or click the area and select the
-file.
-
-
-
-After selecting the file, click on _upload_ to upload it. This usually only takes a few seconds but will take longer in your uploading a large file.
-
-It is also possible to create new folders. To do so, click the "add folder" button
-
just below the project name. An input dialog will be
-presented:
-
-
-
-Fill in the name you want to use and click on the checkmark button
-. Please keep in mind that the folder will only be saved if you put data into it. Select the new folder to select files to upload.
-
-### Resources
-
-All file types can be uploaded into Armadillo, however previews will only be available for `.parquet` files. Other files that can be uploaded are treated as _resources_. Resource filetypes usually are `.rda` files or `.Rdata` files.
-
-To be able to use these resources as a researcher, first an `.rds` file must be generated. How to create these files, is
-described [here](https://molgenis.github.io/molgenis-r-armadillo/articles/create_resources.html).
-
-The URL of your resources should consist of:
-
-```r
-{your url}/storage/projects/{project name}/objects/{name of the folder}%2F{the resource file}
-```
-
-Here is an example, with some example parameters:
-
-```r
-url = "https://armadillo3.demo.molgenis.net"
-project = "omics"
-folder = "ewas"
-file = "gse66351_1.rda"
-```
-
-Which results in the following url:
-
-```r
-https://armadillo3.demo.molgenis.net/storage/projects/omics/objects/ewas%2Fgse66351_1.rda
-```
-
-## Users
-
-The `Users` page works just as the `Projects` page. You can search users by entering (a part of) the email address or name
-of the user into the search box:
-
-
-
-### Editing users
-
-Users can be edited, **except** for their email addresses. This is because when user's email address changes, that user is possibly no longer
-working for the same institution and therefore might not be allowed to access the data anymore.
-
-
-
-In edit mode, the row will turn blue. Projects can be added by clicking on the + icon
- in the projects column.
-
-
-
-You can add a new project by typing it and clicking the checkmark button
-. You will be prompted with a warning message, asking you to confirm if you want to
-add a new project.
-
-Alternatively, you can select an existing project by using the search box or scrolling through the presented list.
-
-### Adding users
-
-By clicking on the plus button on the top of the table, a new user can be
-added. The row with the new user will turn blue in edit mode.
-
-
-
-Users can be added before they have logged in previously. These users can be added to projects, which will grant them
-permission to use the data from those projects upon their first login. Researchers should not be set as admin.
-
-## Analysis Profiles
-
-
-
-Since the release of Armadillo 3.0.0, it is possible to create and manage analysis profiles in the user interface, rather than
-asking system administrators to manage these profiles. You can start 
-and stop 
profiles.
-
-When you start a profile for the first time, it will take a bit longer to load because the profile needs to be
-downloaded and installed before it can be started.
-
-
-
-If you switch to another screen whilst either starting or stopping a profile, the profiles page will no longer show the
-loading information. It is however still loading, and when it's done, if you reload the page, you will see that
-your profile started or stopped successfully.
-
-As in the other screens, you can add profiles with the add-button
-.
-
-
-
-By default, some fields will be set. Please update them to install the correct profile.
-
-Possible images can be found on [dockerhub](https://hub.docker.com/search?q=datashield%2Farmadillo-rserver). We
-recommend selecting one of the
-[DataSHIELD standard profiles](https://wiki.datashield.org/en/opmanag/standard-profiles-and-platforms). The image name of those
-profiles can be found on the dockerhub link above.
-
-Although the default `port` setting should find an available port, please keep in mind that the port has to be unique,
-otherwise you cannot start your profile and will receive an error message.
-
-R packages can be whitelisted by adding them to the `package whitelist` column so researchers can use them. 'Whitelisting' a package allows analysts to use it. If you want
-to whitelist a package, you need to make sure it is installed on the image you selected. Additionally, it is possible to
-blacklist certain R functions in the `blacklist function` column. This can be useful if certain functions
-are not allowed to be used on certain data or within certain cohorts.
diff --git a/docs/v3.4.0/.nojekyll b/docs/v3.4.0/.nojekyll
deleted file mode 100644
index e69de29bb..000000000
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/0_Prerequisite.r b/docs/v3.4.0/DataSHIELD-datamanagement/0_Prerequisite.r
deleted file mode 100644
index fc5d85efd..000000000
--- a/docs/v3.4.0/DataSHIELD-datamanagement/0_Prerequisite.r
+++ /dev/null
@@ -1,19 +0,0 @@
-# install the following packages
-install.packages("MolgenisArmadillo")
-install.packages("dplyr")
-install.packages("arrow")
-# After installation make sure you can load the libraries
-library(MolgenisArmadillo)
-library(dplyr)
-library(arrow)
-# verify the loaded libraries (see: other attached packages)
-sessionInfo()
-
-# optional, install the following packages to be able to run a simple analyses
-# 3_Analyse_data_subset_DSMolgenisArmadillo.r
-install.packages("DSI")
-install.packages("DSMolgenisArmadillo")
-install.packages("dsBaseClient", repos = c("http://cran.datashield.org", "https://cloud.r-project.org/"), dependencies = TRUE)
-
-library(dsBaseClient)
-library(DSMolgenisArmadillo)
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/1_MolgenisArmadillo.r b/docs/v3.4.0/DataSHIELD-datamanagement/1_MolgenisArmadillo.r
deleted file mode 100644
index 78f25bbff..000000000
--- a/docs/v3.4.0/DataSHIELD-datamanagement/1_MolgenisArmadillo.r
+++ /dev/null
@@ -1,119 +0,0 @@
-## Prerequisites
-#
-# - Run the code in 0_Prerequisite.r
-#
-library(MolgenisArmadillo)
-
-##
-# To share your data using Armadillo, you first need to login
-
-## Login
-#
-# In order to access the files as a data manager, you need to log in.
-# The login method needs the URLs of the Armadillo server and the MinIO file server.
-# It will open a browser window where you can identify yourself with the ID provider.
-#
-# If you are unsure how this login function (or any function) works ask R to provide documentation
-?armadillo.login
-
-armadillo.login(
- armadillo = "https://armadillo.test.molgenis.org",
- minio = "https://armadillo-minio.test.molgenis.org"
-)
-#token <- armadillo.get_token("https://armadillo.test.molgenis.org")
-
-# armadillo.login will create a session and store the credentials in the environment.
-
-## Structure
-# To share data via Armadillo you can have a nested structure to save you data.
-
-# We distinguish:
-# - projects
-# - folders
-# - tables
-
-### Projects
-# Projects are root-folders you can give persons permissions on.
-# you can imagine that you will use a separate project for each study you need to support.
-# This way you make sure people can not see each others variables.
-
-### Folders
-# Folder objects can be used to version the different tables you want to share in Armadillo.
-# This is not mandatory and are free to use the folder level as you see fit.
-# In our examples we will go into the versioning part a bit deeper.
-
-### Tables
-# Tables contain the data you want to share.
-# This can be all the data on a certain subject, mostly used in consortia or a specific study you want to expose.
-
-## Sharing data
-# Assume you are in a consortia which has core-variables and outcome-variables.
-# You want to share and version the whole dataset to all researchers which applied to access your data.
-
-# First we will create the project. In our case "ipec".
-
-cohort <- "workshop2"
-armadillo.create_project(cohort)
-
-# Secondly we will load the table(s) we want to upload to Armadillo in the R-environment.
-# We have test data which is in `arrow` format, the upload will take any object that has a table like structure to upload into the Armadillo.
-# This can be SPSS, STATA, SAS or R-based data as well.
-
-library(arrow)
-
-# load the core data (make sure you are working in the correct directory)
-nonrep <- arrow::read_parquet("data/nonrep.parquet")
-yearlyrep <- arrow::read_parquet("data/yearlyrep.parquet")
-
-# explore your data (sanity check)
-View(nonrep)
-dim(nonrep)
-names(nonrep)
-table(nonrep$recruit_age)
-
-# The third step is determining the second level, which contains in this case the datamodel-version the type of variables and the data-version.
-
-# y_y-#variable-type#-x_x
-
-# y_y = datamodel version
-# x_x = data version
-
-# upload the core variables
-armadillo.upload_table(cohort, "2_1-core-1_0", nonrep)
-armadillo.upload_table(cohort, "2_1-core-1_0", yearlyrep)
-
-## Looking at the data
-# There are helper functions to help you determine what is in the storage server.
-# You can list projects and tables to what's in the storage.
-
-# list of projects
-armadillo.list_projects()
-
-# listing tables per project
-armadillo.list_tables(cohort)
-
-# You can download the data in the R-environment as well.
-
-# download table to local R environment
-download_nonrep <- armadillo.load_table(cohort, "2_1-core-1_0", "nonrep")
-
-# check the column names from the local environment
-colnames(download_nonrep)
-
-# check if local data and uploaded data are equal (optional)
-setequal(nonrep, download_nonrep)
-
-# Now you can also take a look at the files in the user interface of the MinIO file server
-# open the MinIO server URL in a browser window (used in armadillo.login).
-
-# > !IMPORTANT: run this part after subsetting the data
-
-## Deleting the data
-# To delete the data you need to throw away the contents first.
-
-# throw away the core tables
-armadillo.delete_table(cohort, "2_1-core-1_0", "nonrep")
-armadillo.delete_table(cohort, "2_1-core-1_0", "yearlyrep")
-
-# Now you can delete the project.
-armadillo.delete_project(cohort)
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/2_Creating_data_subsets_Armadillo.r b/docs/v3.4.0/DataSHIELD-datamanagement/2_Creating_data_subsets_Armadillo.r
deleted file mode 100644
index 0deea0e6b..000000000
--- a/docs/v3.4.0/DataSHIELD-datamanagement/2_Creating_data_subsets_Armadillo.r
+++ /dev/null
@@ -1,96 +0,0 @@
-## Creating data subsets on the Armadillo MiNIO file server
-# When researchers request access to your data they may in many cases not be granted access to the whole data set,
-# but only to a subset. On the MinIO file server access is regulated on the project level,
-# so you will need to create a new project using a subset of the data.
-# Here you can see the different relevant steps you need to take to create these subsets.
-#
-# - Setting up the environment
-# - Logging into the servers
-# - Exploring the data
-# - Making subsets of the data
-# - Upload data subsets
-#
-# - Delete data subsets
-
-## Setting up the environment
-# - Run the code in 0_Prerequisite.r
-
-# load required libraries
-library(MolgenisArmadillo)
-library(dplyr)
-
-## Logging in to the servers
-# In order to access the files on the MinIO fileserver you need to log in using the URLs of the Armadillo server and the MinIO fileserver.
-# A browser window will be opened where you can identify yourself with the ID provider.
-
-armadillo.login(
- armadillo = "https://armadillo.test.molgenis.org",
- minio = "https://armadillo-minio.test.molgenis.org"
-)
-# A session will be created and the credentials are stored in the environment.
-
-## Explore the data
-# Let's assume you are in a consortium which has core-variables and outcome-variables.
-# You want to share a subset of the whole data set with certain researchers that applied for access to your data.
-
-# List projects on the MinIO file server
-armadillo.list_projects()
-
-# Next create a study, here called 'subset1'.
-# NOTE: change the name of the subset for your own practice run
-
-study <- "study1"
-armadillo.create_project(study)
-
-# List the tables in a project
-
-# You want to share data from the cohort you just uploaded (1_MolgenisArmadillo.r).
-# Change the cohort_name to your own cohort name here:
-cohort = "workshop1"
-
-# List the available tables within this project.
-armadillo.list_tables(cohort)
-
-# Subset the core variables
-# Download the relevant core tables to the local environment
-
-nonrep <- armadillo.load_table(cohort, "2_1-core-1_0", "nonrep")
-yearlyrep <- armadillo.load_table(cohort, "2_1-core-1_0", "yearlyrep")
-
-# List their variables
-
-colnames(nonrep)
-colnames(yearlyrep)
-
-# Subset the variables that were requested per table.
-
-subset_core_nonrep <- nonrep %>% select(child_id, asthma_m, preg_cig, preg_fever, preg_alc)
-subset_core_yearlyrep <- yearlyrep %>% select(child_id, cohab_, smk_exp)
-
-## Uploading the data subset
-# Check the variables in the data subset before uploading
-
-colnames(subset_core_nonrep)
-colnames(subset_core_yearlyrep)
-
-# Upload the data subsets
-armadillo.upload_table(study, "2_1-core-1_0", subset_core_nonrep, "nonrep")
-armadillo.upload_table(study, "2_1-core-1_0", subset_core_yearlyrep, "yearlyrep")
-
-# See if tables are uploaded
-armadillo.list_tables(study)
-
-# Now you can also take a look at the files in the user interface of the MinIO file server. In this case: https://armadillo-minio.test.molgenis.org
-
-# > !IMPORTANT: run this part after subsetting the data
-
-## Deleting the data
-# To delete the data you need to throw away the contents first.
-
-# throw away the core tables
-armadillo.delete_table(study, "2_1-core-1_0", "nonrep")
-armadillo.delete_table(study, "2_1-core-1_0", "yearlyrep")
-
-# Now you can delete the project.
-
-armadillo.delete_project(study)
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/3_Analyse_data_subset_DSMolgenisArmadillo.r b/docs/v3.4.0/DataSHIELD-datamanagement/3_Analyse_data_subset_DSMolgenisArmadillo.r
deleted file mode 100644
index f29be2a6d..000000000
--- a/docs/v3.4.0/DataSHIELD-datamanagement/3_Analyse_data_subset_DSMolgenisArmadillo.r
+++ /dev/null
@@ -1,39 +0,0 @@
-install.packages("DSI")
-install.packages("DSMolgenisArmadillo")
-install.packages("dsBaseClient", repos = c("http://cran.datashield.org", "https://cloud.r-project.org/"), dependencies = TRUE)
-
-library(dsBaseClient)
-library(DSMolgenisArmadillo)
-
-# specify server url
-armadillo_url <- "https://armadillo.test.molgenis.org"
-
-# get token from central authentication server
-token <- armadillo.get_token(armadillo_url)
-
-
-# build the login dataframe
-builder <- DSI::newDSLoginBuilder()
-builder$append(
- server = "armadillo",
- url = armadillo_url,
- token = token,
- table = "workshop1/2_1-core-1_0/nonrep",
- driver = "ArmadilloDriver"
-)
-
-# create loginframe
-logindata <- builder$build()
-
-# login into server
-conns <- datashield.login(
- logins = logindata,
- symbol = "core_nonrep",
- variables = c("coh_country"),
- assign = TRUE
-)
-
-# calculate the mean
-ds.mean("core_nonrep$coh_country", datasources = conns)
-
-ds.histogram(x = "core_nonrep$coh_country", datasources = conns)
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/data/nonrep.parquet b/docs/v3.4.0/DataSHIELD-datamanagement/data/nonrep.parquet
deleted file mode 100644
index 7a99ff0d4..000000000
Binary files a/docs/v3.4.0/DataSHIELD-datamanagement/data/nonrep.parquet and /dev/null differ
diff --git a/docs/v3.4.0/DataSHIELD-datamanagement/data/yearlyrep.parquet b/docs/v3.4.0/DataSHIELD-datamanagement/data/yearlyrep.parquet
deleted file mode 100644
index 7049168ed..000000000
Binary files a/docs/v3.4.0/DataSHIELD-datamanagement/data/yearlyrep.parquet and /dev/null differ
diff --git a/docs/v3.4.0/README.md b/docs/v3.4.0/README.md
deleted file mode 100644
index 3352bcfb2..000000000
--- a/docs/v3.4.0/README.md
+++ /dev/null
@@ -1,322 +0,0 @@
-# Armadillo suite
-
-# Overview
-
-Use MOLGENIS/Armadillo to make data available for privacy protecting federated analysis using [DataSHIELD](https://datashield.org) protocol. Armadillo
-service provides the following features:
-* **manage data projects**. Projects can either hold tabular data in the efficient 'parquet' format or any other file use DataSHIELD
- 'resources' framework.
-* **grant users access permission**. We use a central OIDC service like KeyCloak or FusionAuth in combination with a trused identity provider like
- Life Sciences AAI to authenticate users.
-* **configure DataSHIELD analysis profiles**. [DataSHIELD analysis profiles](https://www.datashield.org/help/standard-profiles-and-plaforms) are
- Docker images that contain a collection of multiple [DataSHIELD analysis packages](https://www.datashield.org/help/community-packages).
-
-
-
-# Armadillo installation
-Armadillo requires Java to run, Docker to access the DataSHIELD profiles, and OIDC for authentication (not needed for local tests). Below instructions how to run Armadillo directly from Java, as a Docker container, as a service on Ubuntu or from source code.
-Note that for production you should add a https proxy for essential security. And you might need to enable 'Docker socket' on your docker service.
-
-### Run Armadillo using java commandline
-Software developers often run Armadillo as java jar file:
-
-1. Install Java and Docker (for the DataSHIELD profiles)
-2. Download Armadillo jar file from [releases](https://github.com/molgenis/molgenis-service-armadillo/releases), for example:
-[molgenis-armadillo-3.3.0.jar](https://github.com/molgenis/molgenis-service-armadillo/releases/download/V3.3.0/)
-3. Run armadillo using ```java -jar molgenis-armadillo-3.3.0.jar```
-4. Go to http://localhost:8080 to see your Armadillo running.
-
-Default Armadillo will start with only 'basic-auth' and user 'admin' with password 'admin'. You can enable 'oidc' for connecting more users. You can change
-by providing and editing [application.yaml](application.template.yml) file
-in your working directory and then run command above again.
-
-### Run Armadillo via docker compose
-For testing without having to installing Java you can run using docker:
-
-1. Install [docker-compose](https://docs.docker.com/compose/install/)
-2. Download this [docker-compose.yml](docker-compose.yml).
-3. Execute ```docker-compose up```
-4. Once it says 'Started' go to http://localhost:8080 to see your Armadillo running.
-
-The command must run in same directory as downloaded docker file. We made docker available via 'docker.sock' so we can start/stop DataSHIELD profiles. Alternatively you must include the datashield profiles into this docker-compose. You can override all application.yaml settings via environment variables
-(see commented code in docker-compose file).
-
-### Run Armadillo as service on Ubuntu
-We run Armadillo in production as a Linux service on Ubuntu, ensuring it gets restarted when the server is rebooted. You might be able to reproduce also on
-CentOS (using yum instead of apt).
-
-#### 1. Install necessary software
-```
-apt update
-apt install openjdk-19-jre-headless
-apt install docker.io
-```
-Note: you might need 'sudo'
-
-#### 2. Run installation script
-
-> Download [./scripts/armadillo-setup.sh](./scripts/armadillo-setup.sh) or use it's URL to wget it from the server.
-
-Run wget with URL from above
-```
-wget URL
-```
-This step will install most recent [release](https://github.com/molgenis/molgenis-service-armadillo/releases):
-```
-bash armadillo-setup.sh \
- --admin-user admin \
- --admin-password xxxxx
- --domain my.server.com \
- --oidc \
- --oidc_url https://lifecycle-auth.molgenis.org \
- --oidc_clientid clientid \
- --oidc_clientsecret secret \
- --cleanup
-```
-Note: adapt install command to suit your situation. Use --help to see the options. https://lifecycle-auth.molgenis.org is MOLGENIS provided OIDC service but
-you can also use your own, see FAQ below.
-
-### Running Armadillo from source code
-
-You can run from source code as follows:
-1. Install Java and Docker
-2. Checkout the source using ```git clone https://github.com/molgenis/molgenis-service-armadillo.git```
-3. Optionally copy ```application.template.yml``` to ```application.yml``` to change settings (will be .gitignored)
-4. Compile and execute the code using ```./gradlew run```
-
-Note: contact MOLGENIS team if you want to contribute and need a testing OIDC config that you can run against localhost.
-
-# Using Armadillo
-Armadillo has three main screens to manage projects, user access and DataSHIELD profiles:
-
-### Create data access projects
-Data stewards can use the Armadillo web user interface or [MolgenisArmadillo R client](https://molgenis.github.io/molgenis-r-armadillo)
-to create 'projects' and upload their data into those. Data tables need to be in parquet format that supports fast selections of the columns
-(variables) you need for analysis. Other files can be configured as 'resources'.
-
-### Manage user access
-Data stewards can use the permission screen to give email adresses access to the data. Everybody signs in via single sign on using an OIDC central
-authentication server such as KeyCloack or Fusion auth that federates to authentication systems of
-connected institutions, ideally using a federated AAI such as LifeScience AAI.
-
-### Configure DataSHIELD profiles
-To analyse data, users must choose a datashield profile. Armadillo owners can use the web user interface to configure new profiles. Assuming you
-installed docker you can also start/stop these images. Alternatively you can use docker-compose for that.
-
-There are DataSHIELD packages for [standard statistical analysis](https://github.com/datashield/dsBaseClient)
-, [exposome studies](https://github.com/isglobal-brge/dsExposomeClient)
-, [survival studies](https://github.com/neelsoumya/dsSurvivalClient)
-, [microbiome studies](https://github.com/StuartWheater/dsMicrobiomeClient)
-and [analysis tools for studies that are using large genetic datasets](https://github.com/isglobal-brge/dsomicsclient)
-. These packages can all be installed in the Armadillo suite.
-
-### End users can use Armadillo as any other DataSHIELD server
-A researcher connects from an [R client](https://molgenis.github.io/molgenis-r-datashield) to one or multiple Armadillo servers. The data is
-loaded into an R session on the Armadillo server specifically created for the researcher. Analysis requests are sent to the R session on each Armadillo server.
-There the analysis is performed and aggregated results are sent back to the client.
-
-# Developing Armadillo
-
-We use gradle to build:
-* run using ```./gradlew run```
-* run tests using ```./gradlew test```
-
-We use intellij to develop
-* To run or debug in intellij, right click on armadillo/src/main/java/org.molgenis.armdadillo/ArmadilloServiceAppliction and choose 'Run/Debug Armadillo...'
-* To run using oidc, create a copy of [application.yml](application.template.yml) in root of your project
-
-We have a swagger-ui to quickly see and test available web services at http://localhost:8080/swagger-ui/
-
-# Developing DataSHIELD packages in Armadillo
-As package developer will want to push your new packages into a DataSHIELD profile
-
-* You can start Armadillo with defaults as described above; then use admin/admin as authentication
-* to see what profile are available and has been selected:
-```
-curl -u admin:admin http://localhost:8080/profiles
-```
-* to change selected profile 'my-profile':
-```
-curl -X POST http://localhost:8080/select-profile \
- -H 'Content-Type: application/json' \
- -d 'default'
-```
-* to install-packages in DataSHIELD current using admin user:
-```
-curl -u admin:admin -v \
--H 'Content-Type: multipart/form-data' \
--F "file=@dsBase_6.3.0.tar.gz" \
--X POST http://localhost:8080/install-package
-```
-* to update whitelist of your current profile:
-```
-curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase
-```
-* to get whitelist of current profile:
-```
-curl -u admin:admin http://localhost:8080/whitelist
-```
-
-# Frequently asked questions
-
-### Docker gives a 'java.socket' error
-You might need to enable Docker socket. On Docker desktop you can find that under 'settings' and 'advanced'.
-
-### Can I use docker compose to start profiles?
-Instead of making Armadillo start/stop DataSHIELD profiles you can also use docker compose.
-See commented section in docker-compose.yml file.
-
-### Can I pass environment or commandline variables instead of application.yml?
-
-Yes, it is standard spring.
-
-### Can I run Armadillo with oauth2 config offline?
-Yes, you can run in 'offline' profile
-```
-./gradlew run -Dspring.profiles.active=offline
-```
-
-### How to import data from Armadillo 2?
-To export data from and Armadillo 2 server take the following steps:
-
-#### 1. Install helper software
-```
-apt update
-apt install pip
-pip install minio
-pip install fusionauth-client
-pip install simple_term_menu
-```
-
-#### 2. Backup Armadillo 2 settings
-
-```
-mkdir armadillo2-backup
-rsync -avr /usr/share/armadillo armadillo2-backup
-cp /etc/armadillo/application.yml armadillo2-backup/application-armadillo2.yml
-```
-N.B.change /usr/share to path matching your local config.
-
-##### 3. Export data from Armadillo 2
-This step will copy Armadillo 2 data from minio into the folder matching of an Armadillo 3 data folder:
-
-```
-mkdir data
-wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/v3.4.0/scripts/migrate-minio.py
-python3 migrate-minio.py --minio http://localhost:9000 --target data
-```
-
-N.B.: when aiming running armadillo as a service this folder should be /usr/share/armadillo/data
-
-#### 4. Stop all docker images for Armadillo 2
-List all docker images
-```docker ps -a```
-
-Stop and remove all Armadillo 2 related images, e.g.
-```
-docker rm armadillo_auth_1 armadillo_console_1 armadillo_rserver-default_1 armadillo_rserver-mediation_1 armadillo_rserver-exposome_1 armadillo_rserver-omics_1 armadillo_armadillo_1 -f
-```
-
-#### 5. Remove old minio data
-```rm -Rf /var/lib/minio/ ```
-
-#### 6. Run Armadillo 3 using exported data
-Make sure to move the exported data into the new 'data' folder. Optionally you might need to fix user permissions, e.g.:
-```
-chown armadillo:armadillo -R data
-```
-
-#### 7. Optionally, acquire a permission set from MOLGENIS team
-If you previously ran central authorisation server with MOLGENIS team they can provide you with procedure to load pre-existing permissions.
-They will use:
-```
-wget https://raw.githubusercontent.com/molgenis/molgenis-service-armadillo/v3.4.0/scripts/migrate-auth.py
-python3 migrate-auth.py --fusion-auth https://lifecycle-auth.molgenis.org --armadillo http://localhost:8080
-```
-
-### How to run previous armadillo 2?
-
-For armadillo 2.x you can follow instructions at
-* for testing we use docker compose at https://github.com/molgenis/molgenis-service-armadillo/tree/armadillo-service-2.2.3
-* for production we are using Ansible at https://galaxy.ansible.com/molgenis/armadillo`
-
-### How to run Armadillo as developer?
-
-We develop Armadillo using IntelliJ.
-
-#### To build Armadillo
-To build run following command in the github root:
-```./gradlew build```
-
-To execute in 'dev' run following command in the github root:
-```./gradlew run```
-
-#### Setting up development tools
-
-This repository uses `pre-commit` to manage commit hooks. An installation guide can be found
-[here](https://pre-commit.com/index.html#1-install-pre-commit). To install the hooks, run `pre-commit install` once in the root folder of this repository. Now
-your code will be automatically formatted whenever you commit.
-
-#### How to change data directory
-
-Data is automatically stored in the `data` folder in this repository. You can choose another location
-in `application.yml` by changing the `storage.root-dir`
-setting.
-
-> **_Note_**: When you run Armadillo locally for the first time, the `lifecycle` project has not been
-> added to the system metadata yet. To add it automatically, see [Application properties](#application-properties).
-> Or you can add it manually:
-> - Go to the Swagger UI (`http://localhost:8080/swagger-ui/index.html`)
-> - Go to the `PUT /access/projects` endpoint
-> - Add the project `lifecycle`
->
-> Now you're all set!
-
-#### Working with resources in development mode
-When developing locally, docker has trouble connecting to localhost. This problem becomes clear when working with
-resources. Luckily there's a quick fix for the problem. Instead of defining a resource as for example
-`http://localhost:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda`, rewrite it to:
-`http://host.docker.internal:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda`. Here's some example R code
-for uploading resources:
-```R
-## Uploading resources
-library(MolgenisArmadillo)
-library(resourcer)
-
-token <- armadillo.get_token("http://localhost:8080/")
-
-resGSE1 <- resourcer::newResource(
- name = "GSE66351_1",
- secret = token,
- url = "http://host.docker.internal:8080/storage/projects/omics/objects/test%2Fgse66351_1.rda",
- format = "ExpressionSet"
-)
-
-armadillo.login("http://localhost:8080/")
-armadillo.upload_resource(project="omics", folder="ewas", resource = resGSE1, name = "GSE66351_1")
-```
-And for using them:
-```R
-library(DSMolgenisArmadillo)
-library(dsBaseClient)
-
-token <- armadillo.get_token("http://localhost:8080/")
-
-builder <- DSI::newDSLoginBuilder()
-builder$append(
- server = "local",
- url = "http://localhost:8080/",
- token = token,
- driver = "ArmadilloDriver",
- profile = "uniform",
- resource = "omics/ewas/GSE66351_1"
-)
-
-login_data <- builder$build()
-conns <- DSI::datashield.login(logins = login_data, assign = TRUE)
-
-datashield.resources(conns = conns)
-datashield.assign.resource(conns, resource="omics/ewas/GSE66351_1", symbol="eSet_0y_EUR")
-ds.class('eSet_0y_EUR', datasources = conns)
-datashield.assign.expr(conns, symbol = "methy_0y_EUR",expr = quote(as.resource.object(eSet_0y_EUR)))
-```
diff --git a/docs/v3.4.0/_cdns/docsify-sidebar-collapse.min.js b/docs/v3.4.0/_cdns/docsify-sidebar-collapse.min.js
deleted file mode 100644
index 2b067c7e2..000000000
--- a/docs/v3.4.0/_cdns/docsify-sidebar-collapse.min.js
+++ /dev/null
@@ -1 +0,0 @@
-!function(e){("object"!=typeof exports||"undefined"==typeof module)&&"function"==typeof define&&define.amd?define(e):e()}(function(){"use strict";function e(e,n){var t,a=(n=void 0===n?{}:n).insertAt;e&&"undefined"!=typeof document&&(t=document.head||document.getElementsByTagName("head")[0],(n=document.createElement("style")).type="text/css","top"===a&&t.firstChild?t.insertBefore(n,t.firstChild):t.appendChild(n),n.styleSheet?n.styleSheet.cssText=e:n.appendChild(document.createTextNode(e)))}var t;function a(e){e&&null!=t&&(e=e.getBoundingClientRect().top,document.querySelector(".sidebar").scrollBy(0,e-t))}function n(){requestAnimationFrame(function(){var e=document.querySelector(".app-sub-sidebar > .active");if(e)for(e.parentNode.parentNode.querySelectorAll(".app-sub-sidebar").forEach(function(e){return e.classList.remove("open")});e.parentNode.classList.contains("app-sub-sidebar")&&!e.parentNode.classList.contains("open");)e.parentNode.classList.add("open"),e=e.parentNode})}function o(e){t=e.target.getBoundingClientRect().top;var n=d(e.target,"LI",2);n&&(n.classList.contains("open")?(n.classList.remove("open"),setTimeout(function(){n.classList.add("collapse")},0)):(function(e){if(e)for(e.classList.remove("open","active");e&&"sidebar-nav"!==e.className&&e.parentNode;)"LI"!==e.parentNode.tagName&&"app-sub-sidebar"!==e.parentNode.className||e.parentNode.classList.remove("open"),e=e.parentNode}(s()),i(n),setTimeout(function(){n.classList.remove("collapse")},0)),a(n))}function s(){var e=document.querySelector(".sidebar-nav .active");return e||(e=d(document.querySelector('.sidebar-nav a[href="'.concat(decodeURIComponent(location.hash).replace(/ /gi,"%20"),'"]')),"LI",2))&&e.classList.add("active"),e}function i(e){if(e)for(e.classList.add("open","active");e&&"sidebar-nav"!==e.className&&e.parentNode;)"LI"!==e.parentNode.tagName&&"app-sub-sidebar"!==e.parentNode.className||e.parentNode.classList.add("open"),e=e.parentNode}function d(e,n,t){if(e&&e.tagName===n)return e;for(var a=0;e;){if(t<++a)return;if(e.parentNode.tagName===n)return e.parentNode;e=e.parentNode}}e(".sidebar-nav > ul > li ul {\n display: none;\n}\n\n.app-sub-sidebar {\n display: none;\n}\n\n.app-sub-sidebar.open {\n display: block;\n}\n\n.sidebar-nav .open > ul:not(.app-sub-sidebar),\n.sidebar-nav .active:not(.collapse) > ul {\n display: block;\n}\n\n/* 抖动 */\n.sidebar-nav li.open:not(.collapse) > ul {\n display: block;\n}\n\n.active + ul.app-sub-sidebar {\n display: block;\n}\n"),document.addEventListener("scroll",n);e("@media screen and (max-width: 768px) {\n /* 移动端适配 */\n .markdown-section {\n max-width: none;\n padding: 16px;\n }\n /* 改变原来按钮热区大小 */\n .sidebar-toggle {\n padding: 0 0 10px 10px;\n }\n /* my pin */\n .sidebar-pin {\n appearance: none;\n outline: none;\n position: fixed;\n bottom: 0;\n border: none;\n width: 40px;\n height: 40px;\n background: transparent;\n }\n}\n");var r,c="DOCSIFY_SIDEBAR_PIN_FLAG";function l(){var e="true"===(e=localStorage.getItem(c));localStorage.setItem(c,!e),e?(document.querySelector(".sidebar").style.transform="translateX(0)",document.querySelector(".content").style.transform="translateX(0)"):(document.querySelector(".sidebar").style.transform="translateX(300px)",document.querySelector(".content").style.transform="translateX(300px)")}768'+(i?e:gn(e,!0))+"\n":""+(i?e:gn(e,!0))+"\n"+e+"\n"},e.prototype.html=function(e){return e},e.prototype.heading=function(e,n,i,o){return this.options.headerIds?"
\n":"
\n"},e.prototype.list=function(e,n,i){var o=n?"ol":"ul";return"<"+o+(n&&1!==i?' start="'+i+'"':"")+">\n"+e+"\n"},e.prototype.listitem=function(e){return"
"+e+"
\n"},e.prototype.table=function(e,n){return""+e+""},e.prototype.br=function(){return this.options.xhtml?"":"
"},e.prototype.del=function(e){return"
"+wn(e.message+"",!0)+"";throw e}}xn.options=xn.setOptions=function(e){return bn(xn.defaults,e),yn(xn.defaults),xn},xn.getDefaults=Me,xn.defaults=we,xn.use=function(a){var n,e=bn({},a);if(a.renderer){var i,r=xn.defaults.renderer||new sn;for(i in a.renderer)!function(o){var t=r[o];r[o]=function(){for(var e=[],n=arguments.length;n--;)e[n]=arguments[n];var i=a.renderer[o].apply(r,e);return i=!1===i?t.apply(r,e):i}}(i);e.renderer=r}if(a.tokenizer){var t,c=xn.defaults.tokenizer||new nn;for(t in a.tokenizer)!function(){var o=c[t];c[t]=function(){for(var e=[],n=arguments.length;n--;)e[n]=arguments[n];var i=a.tokenizer[t].apply(c,e);return i=!1===i?o.apply(c,e):i}}();e.tokenizer=c}a.walkTokens&&(n=xn.defaults.walkTokens,e.walkTokens=function(e){a.walkTokens(e),n&&n(e)}),xn.setOptions(e)},xn.walkTokens=function(e,n){for(var i=0,o=e;i
"+wn(e.message+"",!0)+"";throw e}},xn.Parser=mn,xn.parser=mn.parse,xn.Renderer=sn,xn.TextRenderer=ln,xn.Lexer=fn,xn.lexer=fn.lex,xn.Tokenizer=nn,xn.Slugger=vn;var Sn=xn.parse=xn;function An(e,i){if(void 0===i&&(i=''),!e||!e.length)return"";var o="";return e.forEach(function(e){var n=e.title.replace(/(<([^>]+)>)/g,"");o+='
'+n.slice(5).trim()+"
"}function zn(e,o){var t=[],a={};return e.forEach(function(e){var n=e.level||1,i=n-1;o':i;var i,o}).replace(/__colon__/g,":")}function Ln(e){var o={};return{str:e=(e=void 0===e?"":e)&&e.replace(/^('|")/,"").replace(/('|")$/,"").replace(/(?:^|\s):([\w-]+:?)=?([\w-%]+)?/g,function(e,n,i){return-1===n.indexOf(":")?(o[n]=i&&i.replace(/"/g,"")||!0,""):e}).trim(),config:o}}function On(e){return(e=void 0===e?"":e).replace(/(<\/?a.*?>)/gi,"")}var qn,Pn=be(function(e){var u,f,p,d,n,g=function(u){var i=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,n=0,e={},R={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof T?new T(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=r.reach);m+=_.value.length,_=_.next){var b=_.value;if(i.length>n.length)return;if(!(b instanceof T)){var k,w=1;if(l){if(!(k=C(h,m,n,s))||k.index>=n.length)break;var y=k.index,x=k.index+k[0].length,S=m;for(S+=_.value.length;S<=y;)_=_.next,S+=_.value.length;if(S-=_.value.length,m=S,_.value instanceof T)continue;for(var A=_;A!==i.tail&&(S'+Pn.highlight(e.replace(/@DOCSIFY_QM@/g,"`"),i,n)+""+e+"
";return e},t.image=(o=(i={renderer:e,contentBase:o,router:a}).renderer,p=i.contentBase,d=i.router,o.image=function(e,n,i){var o=e,t=[],a=Ln(n),r=a.str,a=a.config;return n=r,a["no-zoom"]&&t.push("data-no-zoom"),n&&t.push('title="'+n+'"'),a.size&&(n=(r=a.size.split("x"))[0],(r=r[1])?t.push('width="'+n+'" height="'+r+'"'):t.push('width="'+n+'"')),a.class&&t.push('class="'+a.class+'"'),a.id&&t.push('id="'+a.id+'"'),R(e)||(o=q(p,C(d.getCurrentPath()),e)),0\x3c!--cover--\x3e
'+e.title+"
\n"+e.content+"
\n\n"}),t.classList.add("show"),a.classList.add("show"),t.innerHTML=r||''+c+"
",s.hideOtherSidebarContent&&(i&&i.classList.add("hide"),n&&n.classList.add("hide"))}function l(e){s=e}function h(e,n){var t,a,i=n.router.parse().query.s;l(e),Docsify.dom.style("\n.sidebar {\n padding-top: 0;\n}\n\n.search {\n margin-bottom: 20px;\n padding: 6px;\n border-bottom: 1px solid #eee;\n}\n\n.search .input-wrap {\n display: flex;\n align-items: center;\n}\n\n.search .results-panel {\n display: none;\n}\n\n.search .results-panel.show {\n display: block;\n}\n\n.search input {\n outline: none;\n border: none;\n width: 100%;\n padding: 0.6em 7px;\n font-size: inherit;\n border: 1px solid transparent;\n}\n\n.search input:focus {\n box-shadow: 0 0 5px var(--theme-color, #42b983);\n border: 1px solid var(--theme-color, #42b983);\n}\n\n.search input::-webkit-search-decoration,\n.search input::-webkit-search-cancel-button,\n.search input {\n -webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n}\n\n.search input::-ms-clear {\n display: none;\n height: 0;\n width: 0;\n}\n\n.search .clear-button {\n cursor: pointer;\n width: 36px;\n text-align: right;\n display: none;\n}\n\n.search .clear-button.show {\n display: block;\n}\n\n.search .clear-button svg {\n transform: scale(.5);\n}\n\n.search h2 {\n font-size: 17px;\n margin: 10px 0;\n}\n\n.search a {\n text-decoration: none;\n color: inherit;\n}\n\n.search .matching-post {\n border-bottom: 1px solid #eee;\n}\n\n.search .matching-post:last-child {\n border-bottom: 0;\n}\n\n.search p {\n font-size: 14px;\n overflow: hidden;\n text-overflow: ellipsis;\n display: -webkit-box;\n -webkit-line-clamp: 2;\n -webkit-box-orient: vertical;\n}\n\n.search p.empty {\n text-align: center;\n}\n\n.app-name.hide, .sidebar-nav.hide {\n display: none;\n}"),function(e){void 0===e&&(e="");var n=Docsify.dom.create("div",'\n \n \n
\n \n '),e=Docsify.dom.find("aside");Docsify.dom.toggleClass(n,"search"),Docsify.dom.before(e,n)}(i),n=Docsify.dom.find("div.search"),a=Docsify.dom.find(n,"input"),e=Docsify.dom.find(n,".input-wrap"),Docsify.dom.on(n,"click",function(e){return-1===["A","H2","P","EM"].indexOf(e.target.tagName)&&e.stopPropagation()}),Docsify.dom.on(a,"input",function(n){clearTimeout(t),t=setTimeout(function(e){return d(n.target.value.trim())},100)}),Docsify.dom.on(e,"click",function(e){"INPUT"!==e.target.tagName&&(a.value="",d())}),i&&setTimeout(function(e){return d(i)},500)}function x(e,n){var t,a,i,r,o;l(e),t=e.placeholder,a=n.route.path,(r=Docsify.dom.getNode('.search input[type="search"]'))&&("string"==typeof t?r.placeholder=t:(i=Object.keys(t).filter(function(e){return-1.
-
-The row will be opened in edit mode:
-
-
-
-The edit mode can be recognized by it's blue background color and you have the option to add new users to your project
-by clicking on the + button of the users column. Then, you can either
-select an existing user from the dropdown, or add the email address of a new user.
-
-
-
-In case of adding a user in this screen, a warning will be shown to prevent email addresses with typographical errors from being added to
-your system. To illustrate this, imagine you have the user `j.doe@example.com` in your users table. You want to give
-them permissions for a certain project, but you mistakingly type the email address as `j.die@example.com`. A warning message will pop up
-asking you whether you really want to add a new user. You will probably think "that's weird, j.doe
-already exists, right?", hopefully this will help noticing the incorrect email address. You will probably cancel
-adding the user and instead select the existing user, as you intended to. Keep in mind that, just as the warning message
-suggests, the user will only be added if you save the row you are editing in the projects.
-
-
-
-It is not possible to edit the name of your project; this was done intentionally in order to ensure tables, resources,
-users, and permissions are transferred to the new project name correctly.
-
-Click on the checkmark to save the edited row and the X
- to cancel. Be careful, if you do cancel your changes will be lost.
-
-### Adding projects
-
-To add a new project, click on the + button on top of the table. If
-you click this button, an empty row will be opened in edit mode.
-
-
-
-Click on the checkmark to save the edited row and the X
- to cancel. Be careful, if you do cancel your changes will be lost.
-
-## Project explorer
-
-If you click on the icon next to the project name, you will be directed to
-the `project editor`. In this screen you can upload and preview data in projects. To go back to your `projects` page,
-press the back button .
-
-
-
-If you click on a folder, it will open.
-
-
-
-Here you can upload files to that folder, or click on the tables (files) to preview their contents.
-
-
-
-To upload files, either drag a file from your file browser to the file upload area, or click the area and select the
-file.
-
-
-
-After selecting the file, click on _upload_ to upload it. Depending on the size of the file it can take a second,
-generally it is a fast process.
-
-It is also possible to create new folders. To do so, click the "add folder" button
- just below the project name. An input dialog will be
-presented:
-
-
-
-Fill in the name you want to use and click on the checkmark button
-. Please keep in mind that, just as the success message suggests,
-the folder will only be saved if you put data in it. Select the new folder to select files to upload.
-
-### Resources
-
-In theory, all files can be uploaded into the projects. The only filetypes that allowed for a preview are `.parquet`
-files. Other files that can be uploaded are treated as _resources_. Resource filetypes usually are `.rda` files or
-`.Rdata` files.
-
-To be able to use these resources as a researcher, first an `.rds` file must be generated. How to create these files, is
-described [here](https://molgenis.github.io/molgenis-r-armadillo/articles/create_resources.html).
-
-The URL of your resources is build up as follows:
-
-```r
-{your url}/storage/projects/{project name}/objects/{name of the folder}%2F{the resource file}
-```
-
-Here's an example, with some example parameters:
-
-```r
-url = "https://armadillo3.demo.molgenis.net"
-project = "omics"
-folder = "ewas"
-file = "gse66351_1.rda"
-```
-
-Which results in the following url:
-
-```r
-https://armadillo3.demo.molgenis.net/storage/projects/omics/objects/ewas%2Fgse66351_1.rda
-```
-
-## Users
-
-The `Users` page works just as the `Projects` page. You can search users by entering (part of) their email address or name
-of the user into the search box:
-
-
-
-### Editing users
-
-Users can be edited, **except** for their email addresses. A reason for this is that when a user's email address changes, that user is possibly not
-working for the same institution any longer and therefore might not be allowed to access the data anymore.
-
-
-
-In edit mode, the row will turn blue. Projects can be added by clicking on the + icon
- in the projects column.
-
-
-
-You can add a new project by simply typing it and clicking the checkmark button
-. You will be prompted a warning message, asking you to confirm to
-add a new project, once the user is saved.
-
-Alternatively, you can select an existing project by using the search box or scrolling through the presented list, and selecting the desired project.
-
-### Adding users
-
-By clicking on the plus button on top op the table, a new user can be
-added. The row with the new user will turn blue in edit mode.
-
-
-
-Users can be added before they have logged in previously. These users can be added to projects, this will grant them
-permission to use the data from those projects upon their first login. However, to be able to login to the UI, users have to be admin or superuser.
-Researchers should not be set to be admin.
-
-## Profiles
-
-
-
-Since the release of Armadillo 3.0.0, it is possible to create and manage profiles in the user interface, rather than
-asking system administrators to manage these profiles. You can start
-and stop profiles.
-
-When you start a profile for the first time, it will take a bit longer to load because the profile needs to be
-downloaded and installed before it can be started.
-
-
-
-If you switch to the other screens while either starting or stopping a profile, the profiles page will not show the
-loading information anymore. It is however still loading, and when it's done, if you reload the page, you will see that
-your profile started or stopped successfully.
-
-Just like in the other screens, you can add profiles with the add-button
-.
-
-
-
-By default, some fields will be set. Please update them to install the correct profile.
-
-Possible images can be found on [dockerhub](https://hub.docker.com/search?q=datashield%2Farmadillo-rserver). We
-recommend selecting one of the
-[DataSHIELD standard profiles](https://www.datashield.org/help/standard-profiles-and-plaforms). The image name of those
-profiles can be found on the dockerhub link above.
-
-Although the default `port` setting should find an available port, please keep in mind that the port has to be unique,
-otherwise you cannot start your profile and will receive and error message.
-
-R packages can be whitelisted by adding them to the `package whitelist` column so researchers can use them. If you want
-to whitelist a package, you need to make sure it is installed on the image you selected. Additionally, it's possible to
-blacklist certain R functions in the `blacklist function` column. This can be interesting to use when certain functions
-are not allowed to be used on certain data or within certain cohorts.
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 000000000..04f5260e6
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,88 @@
+site_name: MOLGENIS Armadillo Docs
+repo_name: MOLGENIS Armadillo
+repo_url: https://github.com/molgenis/molgenis-service-armadillo
+theme:
+ name: material
+ custom_dir: docs/overrides
+ palette:
+ # Palette toggle for light mode
+ - media: "(prefers-color-scheme: light)"
+ scheme: default
+ primary: custom
+ accent: custom
+ toggle:
+ icon: material/brightness-7
+ name: Switch to dark mode
+
+ # Palette toggle for dark mode
+ - media: "(prefers-color-scheme: dark)"
+ scheme: slate
+ primary: custom
+ accent: custom
+ toggle:
+ icon: material/brightness-4
+ name: Switch to light mode
+ favicon: img/favicon.ico
+ logo: img/armadillo-logo.png
+ features:
+ - content.code.copy
+ - content.tooltips
+ - navigation.indexes
+ - navigation.footer
+extra_css:
+ - css/simple.css
+copyright: When using MOLGENIS Armadillo, don't forget to cite us https://doi.org/10.1093/bioinformatics/btae726
+extra:
+ social:
+ - icon: fontawesome/brands/linkedin
+ link: https://www.linkedin.com/company/molgenis
+ - icon: fontawesome/brands/x-twitter
+ link: https://x.com/molgenis
+nav:
+ - "Basic Concepts": pages/basic_concepts.md
+ - "Quick Start" : pages/quick_start.md
+ - "About us": pages/about_us.md
+ - Basic Usage:
+ - pages/basic_usage/index.md
+ - "Armadillo User Interface": pages/basic_usage/armadillo_ui.md
+ - "Central Analysis Server (CAS)": pages/basic_usage/central_analysis_server.md
+ - "DSMolgenisArmadillo": pages/basic_usage/ds_molgenis_armadillo.md
+ - "MolgenisArmadillo": pages/basic_usage/molgenis_armadillo.md
+ - "dsUpload & dsDictionaries": pages/basic_usage/dsupload_dsdictionaries.md
+ - "dsLite": pages/basic_usage/dslite.md
+ - "Authentication Server": pages/basic_usage/auth.md
+# - "Example Usage": pages/examples_usage.md
+# - "Advanced Usage": pages/advanced_usage.md
+ - Install, Setup and Management:
+ - pages/install_management/index.md
+ - "Armadillo Installation": pages/install_management/armadillo_install.md
+ - "Armadillo Management": pages/install_management/armadillo_management.md
+ - "Armadillo Minor Release Update": pages/install_management/armadillo_minor_release_update.md
+ - "Migrate Armadillo 2 to 3": pages/install_management/armadillo_migrate_2_to_3.md
+ - "Migrate Armadillo 3 to 4": pages/install_management/armadillo_migrate_3_to_4.md
+ - "Developer guides": pages/dev_guides.md
+ - "License": pages/license.md
+ - "FAQ": pages/faq.md
+# - "Troubleshooting": pages/troubleshooting.md
+ - "Contact": pages/contact.md
+markdown_extensions:
+ - attr_list
+ - pymdownx.emoji:
+ emoji_index: !!python/name:material.extensions.emoji.twemoji
+ emoji_generator: !!python/name:material.extensions.emoji.to_svg
+ - pymdownx.superfences
+ - pymdownx.tabbed:
+ alternate_style: true
+ - def_list
+ - md_in_html
+ - pymdownx.blocks.caption
+ - pymdownx.highlight:
+ anchor_linenums: true
+ line_spans: __span
+ pygments_lang_class: true
+ - pymdownx.inlinehilite
+ - pymdownx.snippets
+ - admonition
+ - pymdownx.details
+ - abbr
+ - pymdownx.keys
\ No newline at end of file
diff --git a/ui/package.json b/ui/package.json
index 6d74c1f26..3de471bed 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -8,7 +8,7 @@
"preview": "vite preview",
"format": "prettier src --write --config ./.prettierrc.js",
"test": "jest",
- "docs": "docsify serve ../docs"
+ "docs": "mkdocs serve -f ../mkdocs.yml"
},
"dependencies": {
"@popperjs/core": "2.11.8",