Merge pull request #25 from zainasir/mysql-docker

Support pre-loaded data image

.github/workflows/database.yml

@@ -0,0 +1,35 @@
+name: Update Database Image
+
+on:
+  push:
+    paths:
+      - 'data/**'
+
+jobs:
+  update-database-image:
+    runs-on: ubuntu-latest
+    if: github.repository == 'cbioportal/cbioportal-test'
+    env:
+      DOCKER_REPO: 'cbioportal/cbioportal-dev'
+      DOCKER_TAG: 'database'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Log in to DockerHub
+        if: env.CHANGES_DETECTED == 'true'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Update database dump
+        run: |
+          cd $GITHUB_WORKSPACE
+          sh ./scripts/docker-compose.sh --portal_type=web-and-data --docker_args='-d'
+          sh ./utils/check-connection.sh --url=localhost:8080 --interval=5 --max_retries=50
+          sh ./scripts/import-data.sh --seed=./data/seed.sql.gz --schema=./data/cgds.sql --studies=./data/studies
+          sh ./scripts/dump-data.sh --url=localhost:8080 --out=database_dump.sql
+      - name: Push new docker image
+        run: |
+          cd $GITHUB_WORKSPACE
+          docker build -t $DOCKER_REPO:$DOCKER_TAG --build-arg DUMP_PATH=database_dump.sql -f docker/cbioportal-database.Dockerfile .
+          docker image push $DOCKER_REPO:$DOCKER_TAG

.gitignore

@@ -1,3 +1,4 @@
 .idea
 .env
-*metadata.json
+*metadata.json
+**/.DS_Store

README.md

@@ -60,9 +60,23 @@ Import [test data](./data) into a cbioportal instance running at localhost:8080.
 sh ./scripts/import-data.sh --seed=/path/to/seed.sql.gz --schema=/path/to/schema.sql --studies=/path/to/studies-dir
 ```
 
+### [dump-data.sh](./scripts/dump-data.sh)
+Dump the data from cbioportal instance running at given url.
+
+#### Args:
+- _--url=localhost:8080_ (REQUIRED)
+- _--out=/output/path/to/database-dump.sql_ (REQUIRED)
+
+```shell
+sh ./scripts/dump-data.sh --url=localhost:8080 --out=/output/path/to/database-dump.sql
+```
+
 ### Util Scripts
 [utils](./utils) is a collection of scripts that are used in main scripts for improved modularity. See [documentation](./utils/README.md) for usage instructions.
 
+## Docker
+This repository also provides custom Dockerfiles. See [documentation](docker/README.md) for usage instructions.
+
 ## Troubleshoots
 If docker compose gets stuck at "_Database not available yet (first time can take a few minutes to load seed database)... Attempting reconnect..._", try pruning the docker system and rerun script:
 ```shell

data/studies/ascn-test-study/ascn_test_study_data_cna_hg19.seg

@@ -0,0 +1,240 @@
+ID	chrom	loc.start	loc.end	num.mark	seg.mean
+FAKE_P001_S2	1	131450	221346000	2157	-0.3340
+FAKE_P001_S2	1	222641468	224170950	26	-1.9021
+FAKE_P001_S2	1	226064426	247295300	318	-0.4989
+FAKE_P001_S2	2	3862500	39212637	312	0.1666
+FAKE_P001_S2	2	39212800	75175600	452	-0.0008
+FAKE_P001_S2	2	76787826	242982437	1194	0.0535
+FAKE_P001_S2	3	414891	10183150	27	-0.7890
+FAKE_P001_S2	3	10183350	194238700	1904	-0.3191
+FAKE_P001_S2	3	194711896	197916800	159	-0.9094
+FAKE_P001_S2	4	1402525	190662288	1135	-0.3282
+FAKE_P001_S2	5	217988	62836726	614	-0.3405
+FAKE_P001_S2	5	62837000	72743800	62	-0.6818
+FAKE_P001_S2	5	75111183	180722000	889	-0.3196
+FAKE_P001_S2	6	392866	26172300	191	-0.4534
+FAKE_P001_S2	6	26172467	30667938	200	-1.3821
+FAKE_P001_S2	6	30668226	170871300	1201	-0.3438
+FAKE_P001_S2	7	330745	55706275	547	-0.5337
+FAKE_P001_S2	7	55804561	66761640	94	-1.7600
+FAKE_P001_S2	7	66761963	135336256	563	-0.5264
+FAKE_P001_S2	7	135681600	153622306	442	-0.1262
+FAKE_P001_S2	7	153622650	158628150	24	-0.6726
+FAKE_P001_S2	8	564944	146186668	961	-0.0607
+FAKE_P001_S2	9	421700	37034300	518	-0.6439
+FAKE_P001_S2	9	44131135	140406461	770	-0.7108
+FAKE_P001_S2	10	1509750	135181774	801	-0.4233
+FAKE_P001_S2	11	123850	71991850	435	-0.4309
+FAKE_P001_S2	11	71992100	106361902	356	-0.6600
+FAKE_P001_S2	11	106694702	133495950	520	-0.4207
+FAKE_P001_S2	12	394200	133264332	1338	-0.2515
+FAKE_P001_S2	13	19600069	112722650	834	-0.3838
+FAKE_P001_S2	14	19486000	105906100	536	-0.6375
+FAKE_P001_S2	15	20488400	23699617	77	-0.9936
+FAKE_P001_S2	15	26216050	101094335	857	-0.3079
+FAKE_P001_S2	16	114731	34467450	650	-0.3670
+FAKE_P001_S2	16	46387395	46408706	29	3.2591
+FAKE_P001_S2	16	50783300	90242200	661	-0.3476
+FAKE_P001_S2	17	114550	80807828	1822	-0.2988
+FAKE_P001_S2	18	724150	74463750	459	-0.4269
+FAKE_P001_S2	19	1206600	36229639	1107	-0.3378
+FAKE_P001_S2	19	36229700	59096700	346	-0.8340
+FAKE_P001_S2	20	345864	62923750	740	-0.3122
+FAKE_P001_S2	21	10995535	46253150	328	-0.5189
+FAKE_P001_S2	22	16345750	50973133	502	-0.3726
+FAKE_P001_S2	23	451100	152864650	1067	-0.0680
+FAKE_P001_S3	1	131450	247295250	2511	-0.2236
+FAKE_P001_S3	2	3862467	36353139	304	0.2104
+FAKE_P001_S3	2	36353454	69741801	451	-0.0921
+FAKE_P001_S3	2	71183100	242982400	1204	0.2545
+FAKE_P001_S3	3	414891	197916800	2095	-0.2131
+FAKE_P001_S3	4	1402525	89984725	623	-0.2225
+FAKE_P001_S3	4	94653694	99850650	30	-0.9111
+FAKE_P001_S3	4	103804250	190662326	484	-0.1913
+FAKE_P001_S3	5	217988	62836950	615	-0.2105
+FAKE_P001_S3	5	67151500	68390400	54	-0.7689
+FAKE_P001_S3	5	72671550	180722000	897	-0.1708
+FAKE_P001_S3	6	392700	170871300	1599	-0.2511
+FAKE_P001_S3	7	330555	63230350	614	-0.4941
+FAKE_P001_S3	7	63230769	158628217	1061	-0.0399
+FAKE_P001_S3	8	564883	11587450	51	0.2213
+FAKE_P001_S3	8	11856583	41791000	262	-0.1366
+FAKE_P001_S3	8	43828046	146186674	647	0.2109
+FAKE_P001_S3	9	421792	18995323	311	-0.8242
+FAKE_P001_S3	9	18995450	22012422	44	-2.0315
+FAKE_P001_S3	9	26077400	140406350	935	-0.9512
+FAKE_P001_S3	10	1509750	135181776	802	-0.2515
+FAKE_P001_S3	11	123850	76059229	449	-0.2728
+FAKE_P001_S3	11	76059500	121232250	790	-0.8069
+FAKE_P001_S3	11	121232400	133496004	76	-0.1411
+FAKE_P001_S3	12	394200	133264332	1342	-0.1899
+FAKE_P001_S3	13	19600069	112722650	836	-0.5610
+FAKE_P001_S3	14	19486000	105906100	534	-0.7952
+FAKE_P001_S3	15	20488300	101094335	933	-0.2186
+FAKE_P001_S3	16	114913	90242200	1345	-0.2255
+FAKE_P001_S3	17	114500	80807927	1823	-0.1945
+FAKE_P001_S3	18	724000	74463600	462	-0.1986
+FAKE_P001_S3	19	1206596	36229639	1110	-0.2502
+FAKE_P001_S3	19	36229755	59096700	346	-0.9885
+FAKE_P001_S3	20	345864	62923750	742	-0.2100
+FAKE_P001_S3	21	10995575	46253150	328	-0.3029
+FAKE_P001_S3	22	16345900	50973133	503	-0.2277
+FAKE_P001_S3	23	450950	152864550	1066	-0.0393
+FAKE_P001_S4	1	131400	247295250	2484	-0.1628
+FAKE_P001_S4	2	3862500	25472784	110	0.1330
+FAKE_P001_S4	2	25472944	48018410	450	-0.2892
+FAKE_P001_S4	2	48022994	242982437	1380	0.0163
+FAKE_P001_S4	3	414891	197916800	2067	-0.1631
+FAKE_P001_S4	4	1402550	190662319	1122	-0.0979
+FAKE_P001_S4	5	217988	31528784	330	-0.3852
+FAKE_P001_S4	5	31529107	145124092	802	0.0470
+FAKE_P001_S4	5	147496000	180722000	424	-0.2735
+FAKE_P001_S4	6	392750	31240041	427	-0.1368
+FAKE_P001_S4	6	31240293	32821950	140	-0.6306
+FAKE_P001_S4	6	32876982	170871300	1012	-0.1055
+FAKE_P001_S4	7	330745	2883594	19	0.3142
+FAKE_P001_S4	7	2945950	6781627	180	-0.5104
+FAKE_P001_S4	7	6781850	158628200	1451	-0.0818
+FAKE_P001_S4	8	564840	25688548	116	0.0363
+FAKE_P001_S4	8	29194050	38287900	171	-0.4880
+FAKE_P001_S4	8	38297650	146186700	661	-0.0711
+FAKE_P001_S4	9	421800	140406450	1271	-0.3289
+FAKE_P001_S4	10	1509658	135181664	794	-0.1191
+FAKE_P001_S4	11	123950	133495905	1299	-0.2157
+FAKE_P001_S4	12	394421	133264188	1328	-0.2056
+FAKE_P001_S4	13	19600200	112722550	824	-0.1579
+FAKE_P001_S4	14	19485950	105906250	525	-0.1440
+FAKE_P001_S4	15	20488450	101094476	918	-0.1698
+FAKE_P001_S4	16	114800	34417865	633	-0.4192
+FAKE_P001_S4	16	34467200	46408706	31	1.0882
+FAKE_P001_S4	16	50783350	90242200	652	-0.1757
+FAKE_P001_S4	17	114324	80807828	1800	-0.2421
+FAKE_P001_S4	18	724050	74463749	456	-0.0049
+FAKE_P001_S4	19	1206596	59096700	1413	-0.4761
+FAKE_P001_S4	20	345864	62923800	737	-0.1662
+FAKE_P001_S4	21	10995852	46253150	323	-0.2116
+FAKE_P001_S4	22	16345900	50973085	499	-0.2170
+FAKE_P001_S4	23	451000	152864650	1055	-0.1907
+FAKE_P001_S5	1	131350	220441250	2149	-0.4630
+FAKE_P001_S5	1	220441500	224170979	30	-2.0618
+FAKE_P001_S5	1	226064600	247295300	319	-0.4572
+FAKE_P001_S5	2	3862467	36353261	304	0.0301
+FAKE_P001_S5	2	36353454	71183682	455	-0.2911
+FAKE_P001_S5	2	71435350	242982437	1199	-0.0045
+FAKE_P001_S5	3	414891	197916800	2087	-0.4394
+FAKE_P001_S5	4	1402400	106197700	710	-0.3979
+FAKE_P001_S5	4	106197974	142949701	107	-1.1345
+FAKE_P001_S5	4	142949950	190662299	316	-0.2584
+FAKE_P001_S5	5	217988	62836950	615	-0.3603
+FAKE_P001_S5	5	67151346	75926812	64	-0.9731
+FAKE_P001_S5	5	75927274	180722050	881	-0.4314
+FAKE_P001_S5	6	392696	26172200	190	-0.5429
+FAKE_P001_S5	6	26172467	30667850	197	-1.4069
+FAKE_P001_S5	6	30668300	170871300	1200	-0.4704
+FAKE_P001_S5	7	330555	55273457	543	-0.5812
+FAKE_P001_S5	7	55273550	66760400	93	-1.6944
+FAKE_P001_S5	7	66760635	158628200	1033	-0.3784
+FAKE_P001_S5	8	564944	55363754	326	-0.4524
+FAKE_P001_S5	8	55364000	143719983	612	-0.0461
+FAKE_P001_S5	8	144077544	146186668	20	-0.9781
+FAKE_P001_S5	9	421768	18995406	306	-1.0437
+FAKE_P001_S5	9	20776450	27109150	45	-2.3791
+FAKE_P001_S5	9	27109500	37034103	155	-1.1232
+FAKE_P001_S5	9	44131135	139820750	763	-1.2859
+FAKE_P001_S5	10	1509750	135181750	799	-0.5199
+FAKE_P001_S5	11	123900	71991687	434	-0.5398
+FAKE_P001_S5	11	71992084	120491250	794	-1.0122
+FAKE_P001_S5	11	121232296	133496000	77	-0.4858
+FAKE_P001_S5	12	394213	133264332	1337	-0.3785
+FAKE_P001_S5	13	19600126	112722600	834	-0.6211
+FAKE_P001_S5	14	19486250	105906250	529	-1.0947
+FAKE_P001_S5	15	20488400	101094476	933	-0.4532
+FAKE_P001_S5	16	114933	34467250	648	-0.4505
+FAKE_P001_S5	16	46387400	46408706	29	3.1865
+FAKE_P001_S5	16	50783300	90242200	665	-0.4417
+FAKE_P001_S5	17	114530	80807912	1821	-0.4103
+FAKE_P001_S5	18	724000	74463650	460	-0.4772
+FAKE_P001_S5	19	1206700	36229755	1109	-0.4357
+FAKE_P001_S5	19	36833464	59096750	341	-1.2524
+FAKE_P001_S5	20	345864	62923800	741	-0.3777
+FAKE_P001_S5	21	10995535	46253300	328	-0.5384
+FAKE_P001_S5	22	16345800	50973133	501	-0.4544
+FAKE_P001_S5	23	451000	39911200	123	-0.3380
+FAKE_P001_S5	23	39911400	71788885	498	0.0610
+FAKE_P001_S5	23	71789100	152864600	444	-0.3133
+FAKE_P001_S6	1	131500	247295200	2511	-0.3881
+FAKE_P001_S6	2	3862450	29940600	294	0.0143
+FAKE_P001_S6	2	29940921	71435350	466	-0.2708
+FAKE_P001_S6	2	72126350	242982400	1199	0.1393
+FAKE_P001_S6	3	414891	197916800	2094	-0.3631
+FAKE_P001_S6	4	1402594	190662288	1135	-0.3499
+FAKE_P001_S6	5	217988	57754808	594	-0.2980
+FAKE_P001_S6	5	57755050	79967900	113	-0.8251
+FAKE_P001_S6	5	79968241	180722000	859	-0.3061
+FAKE_P001_S6	6	392783	170871300	1597	-0.3426
+FAKE_P001_S6	7	330700	158628217	1673	-0.2856
+FAKE_P001_S6	8	564944	12586261	56	0.0243
+FAKE_P001_S6	8	12586593	41791015	258	-0.3208
+FAKE_P001_S6	8	43828046	146186692	646	0.1191
+FAKE_P001_S6	9	421768	18995379	311	-0.8638
+FAKE_P001_S6	9	20776484	22012422	43	-1.8816
+FAKE_P001_S6	9	26077500	27220272	88	-0.7752
+FAKE_P001_S6	9	27220550	140406350	844	-0.9909
+FAKE_P001_S6	10	1509800	135181630	802	-0.4182
+FAKE_P001_S6	11	123900	74790200	445	-0.4552
+FAKE_P001_S6	11	74790400	121232100	789	-0.7177
+FAKE_P001_S6	11	121232350	133496004	76	-0.2076
+FAKE_P001_S6	12	394250	133264332	1342	-0.2468
+FAKE_P001_S6	13	19600126	112722700	834	-0.7743
+FAKE_P001_S6	14	19486000	105906050	534	-0.8490
+FAKE_P001_S6	15	20488250	101094335	933	-0.3441
+FAKE_P001_S6	16	114727	34467500	650	-0.5065
+FAKE_P001_S6	16	46387397	46408706	29	0.6443
+FAKE_P001_S6	16	50783350	90242200	664	-0.4356
+FAKE_P001_S6	17	114400	80807897	1823	-0.4005
+FAKE_P001_S6	18	724107	74463600	462	-0.3526
+FAKE_P001_S6	19	1206600	36229600	1109	-0.4905
+FAKE_P001_S6	19	36229850	59096700	347	-1.0935
+FAKE_P001_S6	20	345864	62923800	742	-0.3527
+FAKE_P001_S6	21	10995575	46253250	328	-0.4249
+FAKE_P001_S6	22	16345938	50973085	503	-0.4012
+FAKE_P001_S6	23	450950	152864600	1067	-0.0636
+FAKE_P001_S7	1	131350	221345796	2159	-0.3894
+FAKE_P001_S7	1	221346050	224170950	27	-1.9645
+FAKE_P001_S7	1	226064450	247295350	318	-0.4560
+FAKE_P001_S7	2	3862550	242982400	1959	-0.3671
+FAKE_P001_S7	3	414891	10187982	35	-0.7602
+FAKE_P001_S7	3	10188229	194238717	1895	-0.3091
+FAKE_P001_S7	3	194711850	197916800	159	-0.9004
+FAKE_P001_S7	4	1402580	106197850	709	-0.3438
+FAKE_P001_S7	4	106197950	142737548	106	-1.2143
+FAKE_P001_S7	4	142949750	190662299	318	-0.1790
+FAKE_P001_S7	5	217988	180722100	1564	-0.3240
+FAKE_P001_S7	6	392866	26172300	193	-0.4888
+FAKE_P001_S7	6	26172467	30460500	198	-1.3912
+FAKE_P001_S7	6	30668071	55672900	340	-0.4405
+FAKE_P001_S7	6	58776622	58779088	7	3.0639
+FAKE_P001_S7	6	64286500	170871300	857	-0.3159
+FAKE_P001_S7	7	330550	55273306	543	-0.4693
+FAKE_P001_S7	7	55273550	66761667	99	-1.5772
+FAKE_P001_S7	7	66762080	158628050	1029	-0.3193
+FAKE_P001_S7	8	564883	146186750	960	-0.4127
+FAKE_P001_S7	9	421600	139820800	1285	-0.2825
+FAKE_P001_S7	10	1509770	135181631	800	-0.4850
+FAKE_P001_S7	11	123850	133495905	1313	-0.3496
+FAKE_P001_S7	12	394650	133264332	1339	-0.2729
+FAKE_P001_S7	13	19600069	112722600	836	-0.3231
+FAKE_P001_S7	14	19485950	105906200	536	-0.3680
+FAKE_P001_S7	15	20488350	101094335	934	-0.4018
+FAKE_P001_S7	16	114937	34467500	650	-0.3702
+FAKE_P001_S7	16	46387437	46408706	29	3.1368
+FAKE_P001_S7	16	50783306	90242200	660	-0.3016
+FAKE_P001_S7	17	114518	80807912	1820	-0.3283
+FAKE_P001_S7	18	724224	74463692	462	-0.4292
+FAKE_P001_S7	19	1206550	59096700	1455	-0.3923
+FAKE_P001_S7	20	345864	62923800	741	-0.2813
+FAKE_P001_S7	21	10995535	46253250	328	-0.4404
+FAKE_P001_S7	22	16345855	50973133	502	-0.4264
+FAKE_P001_S7	23	451050	32224384	122	-0.1871
+FAKE_P001_S7	23	39911300	70357900	487	0.0957
+FAKE_P001_S7	23	70358205	152864500	457	-0.2397

data/studies/ascn-test-study/ascn_test_study_meta_cna_hg19_seg.txt

@@ -0,0 +1,6 @@
+cancer_study_identifier: ascn_test_study
+data_filename: ascn_test_study_data_cna_hg19.seg
+datatype: SEG
+description: Segmented Data
+genetic_alteration_type: COPY_NUMBER_ALTERATION
+reference_genome_id: hg19

data/studies/ascn-test-study/case_lists/cases_all.txt

@@ -0,0 +1,6 @@
+cancer_study_identifier: ascn_test_study
+stable_id: ascn_test_study_all
+case_list_category: all_cases_in_study
+case_list_name: All Tumors
+case_list_description: All tumor samples
+case_list_ids: FAKE_P001_S2	FAKE_P001_S3	FAKE_P001_S4	FAKE_P001_S5	FAKE_P001_S6	FAKE_P001_S7

data/studies/ascn-test-study/case_lists/cases_cna.txt

@@ -0,0 +1,6 @@
+cancer_study_identifier: ascn_test_study
+stable_id: ascn_test_study_cna
+case_list_category: all_cases_with_cna_data
+case_list_name: Tumors CNA
+case_list_description: All tumors with CNA data
+case_list_ids: FAKE_P001_S2	FAKE_P001_S3	FAKE_P001_S4	FAKE_P001_S5	FAKE_P001_S6	FAKE_P001_S7

data/studies/ascn-test-study/case_lists/cases_cnaseq.txt

@@ -0,0 +1,6 @@
+cancer_study_identifier: ascn_test_study
+stable_id: ascn_test_study_cnaseq
+case_list_category: all_cases_with_mutation_and_cna_data
+case_list_name: Tumors with sequencing and CNA data
+case_list_description: All tumor samples that have CNA and sequencing data
+case_list_ids: FAKE_P001_S2	FAKE_P001_S3	FAKE_P001_S4	FAKE_P001_S5	FAKE_P001_S6

data/studies/ascn-test-study/case_lists/cases_sequenced.txt

@@ -0,0 +1,6 @@
+cancer_study_identifier: ascn_test_study
+stable_id: ascn_test_study_sequenced
+case_list_category: all_cases_with_mutation_data
+case_list_name: Sequenced Tumors
+case_list_description: All sequenced tumors
+case_list_ids: FAKE_P001_S2	FAKE_P001_S3	FAKE_P001_S4	FAKE_P001_S5	FAKE_P001_S6