diff --git a/src/airflow/dags/gwas_curation_update.py b/src/airflow/dags/gwas_curation_update.py index 1ef0f39f9..830007e6d 100644 --- a/src/airflow/dags/gwas_curation_update.py +++ b/src/airflow/dags/gwas_curation_update.py @@ -18,7 +18,7 @@ ): update_gwas_curation = common.submit_step( cluster_name=CLUSTER_NAME, - step_id="gwas_catalog_curation_update", + step_id="ot_gwas_catalog_study_curation", task_id="gwas_catalog_curation_update", other_args=[ f"step.gwas_catalog_study_curation_out=gs://genetics_etl_python_playground/input/v2d/GWAS_Catalog_study_curation_{RUN_DATE}.tsv", diff --git a/src/gentropy/datasource/gwas_catalog/study_index.py b/src/gentropy/datasource/gwas_catalog/study_index.py index cb6d3338a..e8b49b0da 100644 --- a/src/gentropy/datasource/gwas_catalog/study_index.py +++ b/src/gentropy/datasource/gwas_catalog/study_index.py @@ -305,6 +305,7 @@ def _parse_study_table( parse_efos(f.col("MAPPED BACKGROUND TRAIT URI")).alias( "backgroundTraitFromSourceMappedIds" ), + cls.parse_cohorts(f.col("COHORT")).alias("cohorts"), ), _schema=StudyIndexGWASCatalog.get_schema(), ) @@ -548,14 +549,6 @@ def annotate_ancestries( ) # studyId has not been split yet ) - # Parsing cohort information: - cohorts = ancestry_lut.select( - f.col("STUDY ACCESSION").alias("studyId"), - GWASCatalogStudyIndexParser.parse_cohorts(f.col("COHORT(S)")).alias( - "cohorts" - ), - ).distinct() - # Get a high resolution dataset on experimental stage: ancestry_stages = ( ancestry.groupBy("studyId") @@ -644,9 +637,7 @@ def annotate_ancestries( ).select( "studyId", "discoverySamples", "ldPopulationStructure", "replicationSamples" ) - self.df = self.df.join(parsed_ancestry_lut, on="studyId", how="left").join( - cohorts, on="studyId", how="left" - ) + self.df = self.df.join(parsed_ancestry_lut, on="studyId", how="left") return self def annotate_sumstats_info( diff --git a/tests/gentropy/conftest.py b/tests/gentropy/conftest.py index 13409ac16..78bd567da 100644 --- a/tests/gentropy/conftest.py +++ b/tests/gentropy/conftest.py @@ -414,7 +414,7 @@ def mock_ld_index(spark: SparkSession) -> LDIndex: def sample_gwas_catalog_studies(spark: SparkSession) -> DataFrame: """Sample GWAS Catalog studies.""" return spark.read.csv( - "tests/gentropy/data_samples/gwas_catalog_studies_sample-r2022-11-29.tsv", + "tests/gentropy/data_samples/gwas_catalog_studies.tsv", sep="\t", header=True, ) @@ -424,7 +424,7 @@ def sample_gwas_catalog_studies(spark: SparkSession) -> DataFrame: def sample_gwas_catalog_ancestries_lut(spark: SparkSession) -> DataFrame: """Sample GWAS ancestries sample data.""" return spark.read.csv( - "tests/gentropy/data_samples/gwas_catalog_ancestries_sample_v1.0.3-r2022-11-29.tsv", + "tests/gentropy/data_samples/gwas_catalog_ancestries.tsv", sep="\t", header=True, ) @@ -444,7 +444,7 @@ def sample_gwas_catalog_harmonised_sumstats_list(spark: SparkSession) -> DataFra def sample_gwas_catalog_associations(spark: SparkSession) -> DataFrame: """Sample GWAS raw associations sample data.""" return spark.read.csv( - "tests/gentropy/data_samples/gwas_catalog_associations_sample_e107_r2022-11-29.tsv", + "tests/gentropy/data_samples/gwas_catalog_associations.tsv", sep="\t", header=True, ) diff --git a/tests/gentropy/data_samples/gwas_catalog_ancestries.tsv b/tests/gentropy/data_samples/gwas_catalog_ancestries.tsv new file mode 100644 index 000000000..48423eeee --- /dev/null +++ b/tests/gentropy/data_samples/gwas_catalog_ancestries.tsv @@ -0,0 +1,20 @@ +STUDY ACCESSION PUBMED ID FIRST AUTHOR DATE INITIAL SAMPLE DESCRIPTION REPLICATION SAMPLE DESCRIPTION STAGE NUMBER OF INDIVIDUALS BROAD ANCESTRAL CATEGORY COUNTRY OF ORIGIN COUNTRY OF RECRUITMENT ADDITIONAL ANCESTRY DESCRIPTION ANCESTRY DESCRIPTOR FOUNDER/GENETICALLY ISOLATED POPULATION NUMBER OF CASES NUMBER OF CONTROLS SAMPLE DESCRIPTION +GCST004795 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 205 Asian unspecified, African American or Afro-Caribbean, Native American, Other admixed ancestry NR U.S. +GCST004795 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 356 European NR U.S. +GCST004796 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 356 European NR U.S. +GCST004796 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 205 Asian unspecified, African American or Afro-Caribbean, Native American, Other admixed ancestry NR U.S. +GCST004797 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 205 Asian unspecified, African American or Afro-Caribbean, Native American, Other admixed ancestry NR U.S. +GCST004797 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 356 European NR U.S. +GCST004794 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 356 European NR U.S. +GCST004794 28763065 Xia K 2017-08-01 295 European, African American, Asian, Native American and other admixed ancestry infants, 17 European, African American, Asian, Native American and other admixed ancestry sibling pairs, 116 European, African American, Asian, Native American and other admixed ancestry twin pairs. NA initial 205 Asian unspecified, African American or Afro-Caribbean, Native American, Other admixed ancestry NR U.S. +GCST005522 23459209 Faraco J 2013-01-01 1,886 European ancestry cases, 10,421 European ancestry controls NA initial 12307 European NR Canada, U.S., Australia, Austria, France, Germany, Netherlands, Switzerland, Argentina, Israel, Turkey, Czech Republic, Poland, Slovakia, Denmark, Finland, Norway, U.K., Italy, Portugal, Spain +GCST004692 27455348 van Rheenen W 2016-07-25 12,577 European ancestry cases, 23,475 European ancestry controls 2,579 European ancestry cases, 2,767 European ancestry controls initial 36052 European NR U.S., Belgium, France, Germany, Netherlands, Switzerland, Finland, Republic of Ireland, Sweden, U.K., Italy, Portugal, Spain +GCST004692 27455348 van Rheenen W 2016-07-25 12,577 European ancestry cases, 23,475 European ancestry controls 2,579 European ancestry cases, 2,767 European ancestry controls replication 5346 European NR Australia, Belgium, France, Germany, Netherlands, Turkey, Republic of Ireland, Italy +GCST005134 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals replication 64 Greater Middle Eastern (Middle Eastern, North African or Persian) NR France +GCST005134 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals replication 431 European NR France +GCST005134 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals initial 448 European NR France +GCST005134 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals initial 47 Greater Middle Eastern (Middle Eastern, North African or Persian) NR France +GCST005135 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals initial 448 European NR France +GCST005135 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals initial 47 Greater Middle Eastern (Middle Eastern, North African or Persian) NR France +GCST005135 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals replication 431 European NR France +GCST005135 28800628 Salem JE 2017-08-11 448 European ancestry individuals, 47 North African ancestry individuals 431 European ancestry individuals, 64 North African ancestry individuals replication 64 Greater Middle Eastern (Middle Eastern, North African or Persian) NR France diff --git a/tests/gentropy/data_samples/gwas_catalog_ancestries_sample_v1.0.3-r2022-11-29.tsv b/tests/gentropy/data_samples/gwas_catalog_ancestries_sample_v1.0.3-r2022-11-29.tsv deleted file mode 100644 index 92d87d92b..000000000 --- a/tests/gentropy/data_samples/gwas_catalog_ancestries_sample_v1.0.3-r2022-11-29.tsv +++ /dev/null @@ -1,20 +0,0 @@ -STUDY ACCESSION PUBMED ID FIRST AUTHOR DATE INITIAL SAMPLE DESCRIPTION REPLICATION SAMPLE DESCRIPTION STAGE NUMBER OF INDIVIDUALS BROAD ANCESTRAL CATEGORY COUNTRY OF ORIGIN COUNTRY OF RECRUITMENT ADDITIONAL ANCESTRY DESCRIPTION ANCESTRY DESCRIPTOR FOUNDER/GENETICALLY ISOLATED POPULATION NUMBER OF CASES NUMBER OF CONTROLS SAMPLE DESCRIPTION COHORT(S) COHORT-SPECIFIC REFERENCE -GCST008644 26546613 Gutierrez-Achury J 2016-01-01 371 South Asian ancestry celiac disease cases, 3,138 European ancestry celiac disease cases, 4,418 European ancestry rheumatoid arthritis cases, 509 South Asian ancestry celiac disease controls, 2,473 European ancestry celiac disease controls, 3,300 European ancestry rheumatoid arthritis controls, 8,872 celiac disease cases, 9,401 rheumatoid arthritis cases, 4,845 celiac disease controls, 9,627 rheumatoid arthritis controls NA initial 32475 NR NR U.S., Netherlands, U.K. -GCST008644 26546613 Gutierrez-Achury J 2016-01-01 371 South Asian ancestry celiac disease cases, 3,138 European ancestry celiac disease cases, 4,418 European ancestry rheumatoid arthritis cases, 509 South Asian ancestry celiac disease controls, 2,473 European ancestry celiac disease controls, 3,300 European ancestry rheumatoid arthritis controls, 8,872 celiac disease cases, 9,401 rheumatoid arthritis cases, 4,845 celiac disease controls, 9,627 rheumatoid arthritis controls NA initial 13329 European NR Sweden, Poland, Italy, Spain -GCST008644 26546613 Gutierrez-Achury J 2016-01-01 371 South Asian ancestry celiac disease cases, 3,138 European ancestry celiac disease cases, 4,418 European ancestry rheumatoid arthritis cases, 509 South Asian ancestry celiac disease controls, 2,473 European ancestry celiac disease controls, 3,300 European ancestry rheumatoid arthritis controls, 8,872 celiac disease cases, 9,401 rheumatoid arthritis cases, 4,845 celiac disease controls, 9,627 rheumatoid arthritis controls NA initial 880 South Asian NR India -GCST004026 27911795 Schumann G 2016-11-28 up to 70,460 European ancestry drinker individuals up to 35,438 European ancestry drinker individuals replication 35438 European NR Finland, Sweden, Italy, Netherlands, U.K., Austria, France, Republic of Ireland -GCST004026 27911795 Schumann G 2016-11-28 up to 70,460 European ancestry drinker individuals up to 35,438 European ancestry drinker individuals initial 70460 European NR Finland, U.S., Australia, Iceland, Netherlands, Germany, U.K., Switzerland, Estonia, NR -GCST004027 27911795 Schumann G 2016-11-28 up to 74,711 European ancestry heavy and light/non-drinker individuals up to 31,021 European ancestry heavy and light/non-drinker individuals initial 74711 European NR Finland, U.S., Australia, Iceland, Netherlands, Germany, U.K., Switzerland, Estonia, NR, France -GCST004027 27911795 Schumann G 2016-11-28 up to 74,711 European ancestry heavy and light/non-drinker individuals up to 31,021 European ancestry heavy and light/non-drinker individuals replication 31021 European NR Finland, U.S., Italy, Netherlands, U.K., Austria, Republic of Ireland -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 23 South Asian NR U.S. -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 329 Hispanic or Latin American NR U.S. -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 21 African American or Afro-Caribbean NR U.S. -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 273 European NR U.S. -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 22 Other NR U.S. -GCST004281 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 122 Asian unspecified NR U.S. -GCST004284 28235828 Traglia M 2017-04-03 764 fetuses NA initial 764 European, South Asian, Asian unspecified, African American or Afro-Caribbean, Hispanic or Latin American, Other NR U.S. -GCST004285 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 329 Hispanic or Latin American NR U.S. -GCST004285 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 273 European NR U.S. -GCST004285 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 23 South Asian NR U.S. -GCST004285 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 21 African American or Afro-Caribbean NR U.S. -GCST004285 28235828 Traglia M 2017-04-03 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA initial 22 Other NR U.S. diff --git a/tests/gentropy/data_samples/gwas_catalog_associations_sample_e107_r2022-11-29.tsv b/tests/gentropy/data_samples/gwas_catalog_associations.tsv similarity index 100% rename from tests/gentropy/data_samples/gwas_catalog_associations_sample_e107_r2022-11-29.tsv rename to tests/gentropy/data_samples/gwas_catalog_associations.tsv diff --git a/tests/gentropy/data_samples/gwas_catalog_studies.tsv b/tests/gentropy/data_samples/gwas_catalog_studies.tsv new file mode 100644 index 000000000..aca9ed666 --- /dev/null +++ b/tests/gentropy/data_samples/gwas_catalog_studies.tsv @@ -0,0 +1,20 @@ +DATE ADDED TO CATALOG PUBMED ID FIRST AUTHOR DATE JOURNAL LINK STUDY DISEASE/TRAIT INITIAL SAMPLE SIZE REPLICATION SAMPLE SIZE PLATFORM [SNPS PASSING QC] ASSOCIATION COUNT MAPPED_TRAIT MAPPED_TRAIT_URI STUDY ACCESSION GENOTYPING TECHNOLOGY SUBMISSION DATE STATISTICAL MODEL BACKGROUND TRAIT MAPPED BACKGROUND TRAIT MAPPED BACKGROUND TRAIT URI COHORT FULL SUMMARY STATISTICS SUMMARY STATS LOCATION +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints 12,863 European ancestry male cases, 19,521 European ancestry female cases, 40,776 European ancestry male controls, 39,846 European ancestry female controls 1,983 Icelandic ancestry male cases, 1,791 Icelandic ancestry female cases, 2,064 Icelandic ancestry male controls, 1,727 Icelandic ancestry female controls Affymetrix [at least 12428592] (imputed) 2 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST004695 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004695 +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Sleep duration 112,411 European ancestry male individuals NA Affymetrix [at least 12428592] (imputed) 3 sleep duration http://www.ebi.ac.uk/efo/EFO_0005271 GCST004694 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints (sex interaction) 12,863 European ancestry male cases, 19,521 European ancestry female cases, 40,776 European ancestry male controls, 39,846 European ancestry female controls NA Affymetrix [at least 12428592] (imputed) 0 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST004700 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints (continuous) 12,863 European ancestry male cases, 19,521 European ancestry female cases, 40,776 European ancestry male controls, 39,846 European ancestry female controls NA Affymetrix [at least 12428592] (imputed) 1 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST004701 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints (dichotomous) 32,384 European ancestry cases, 27,128 European ancestry controls NA Affymetrix [at least 12428592] (imputed) 1 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST004702 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Chronotype 101,185 European ancestry individuals NA Affymetrix [at least 12428592] (imputed) 9 circadian rhythm http://www.ebi.ac.uk/efo/EFO_0004354 GCST004696 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Ease of getting up in the morning 112,866 European ancestry individuals NA Affymetrix [at least 12428592] (imputed) 5 chronotype measurement http://www.ebi.ac.uk/efo/EFO_0008328 GCST004697 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Daytime nap 113,054 European ancestry cases and controls NA Affymetrix [at least 12428592] (imputed) 4 daytime rest measurement http://www.ebi.ac.uk/efo/EFO_0007828 GCST004693 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Snoring 105,377 European ancestry cases and controls NA Affymetrix [at least 12428592] (imputed) 1 snoring measurement http://www.ebi.ac.uk/efo/EFO_0008341 GCST004698 Genome-wide genotyping array no NA +2017-09-11 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Daytime sleepiness 112,717 European ancestry cases and controls NA Affymetrix [at least 12428592] (imputed) 6 excessive daytime sleepiness measurement http://www.ebi.ac.uk/efo/EFO_0007875 GCST004699 Genome-wide genotyping array no NA +2018-11-02 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints 19,521 European ancestry female cases, 39,846 European ancestry female controls 1,983 Icelandic ancestry male cases, 1,791 Icelandic ancestry female cases, 2,064 Icelandic ancestry male controls, 1,727 Icelandic ancestry female controls Affymetrix [at least 12428592] (imputed) 1 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST006487 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST006001-GCST007000/GCST006487 +2018-11-02 28604731 Hammerschlag AR 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604731 Genome-wide association analysis of insomnia complaints identifies risk genes and genetic overlap with psychiatric and metabolic traits. Insomnia complaints 12,863 European ancestry male cases, 40,776 European ancestry male controls 1,983 Icelandic ancestry male cases, 1,791 Icelandic ancestry female cases, 2,064 Icelandic ancestry male controls, 1,727 Icelandic ancestry female controls Affymetrix [at least 12428592] (imputed) 2 insomnia http://www.ebi.ac.uk/efo/EFO_0004698 GCST006488 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST006001-GCST007000/GCST006488 +2017-09-14 28610988 Kerr KF 2017-06-10 Heart Rhythm www.ncbi.nlm.nih.gov/pubmed/28610988 Genome-wide association study of heart rate and its variability in Hispanic/Latino cohorts. Heart rate 13,184 Hispanic/Latino individuals 7,073 European ancestry individuals, 4,771 African American individuals Affymetrix, Illumina [16967914] (imputed) 2 heart rate http://www.ebi.ac.uk/efo/EFO_0004326 GCST004715 Genome-wide genotyping array no NA +2017-09-14 28610988 Kerr KF 2017-06-10 Heart Rhythm www.ncbi.nlm.nih.gov/pubmed/28610988 Genome-wide association study of heart rate and its variability in Hispanic/Latino cohorts. Heart rate variability traits (RMSSD) 13,767 Hispanic/Latino individuals 4,730 European ancestry individuals, 2,908 African American individuals Affymetrix, Illumina [17209892] (imputed) 2 heart rate variability measurement http://www.ebi.ac.uk/efo/EFO_0008003 GCST004716 Genome-wide genotyping array no NA +2017-09-14 28610988 Kerr KF 2017-06-10 Heart Rhythm www.ncbi.nlm.nih.gov/pubmed/28610988 Genome-wide association study of heart rate and its variability in Hispanic/Latino cohorts. Heart rate variability traits (SDNN) 13,184 Hispanic/Latino individuals 7,073 European ancestry individuals, 2,908 African American individuals Affymetrix, Illumina [17209740] (imputed) 3 heart rate variability measurement http://www.ebi.ac.uk/efo/EFO_0008003 GCST004714 Genome-wide genotyping array no NA +2017-09-18 28604730 McKay JD 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604730 Large-scale association analysis identifies new lung cancer susceptibility loci and heterogeneity in genetic susceptibility across histological subtypes. Lung cancer 29,266 European ancestry cases, 56,450 European ancestry controls NA Illumina [10439017] (imputed) 135 lung carcinoma http://www.ebi.ac.uk/efo/EFO_0001071 GCST004748 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004748 +2017-09-18 28604730 McKay JD 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604730 Large-scale association analysis identifies new lung cancer susceptibility loci and heterogeneity in genetic susceptibility across histological subtypes. Lung adenocarcinoma 11,273 European ancestry cases, 55,483 European ancestry controls NA Illumina [10439017] (imputed) 79 lung adenocarcinoma http://www.ebi.ac.uk/efo/EFO_0000571 GCST004744 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004744 +2017-09-18 28604730 McKay JD 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604730 Large-scale association analysis identifies new lung cancer susceptibility loci and heterogeneity in genetic susceptibility across histological subtypes. Squamous cell lung carcinoma 7,426 European ancestry cases, 55,627 European ancestry controls NA Illumina [10439017] (imputed) 101 squamous cell lung carcinoma http://www.ebi.ac.uk/efo/EFO_0000708 GCST004750 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004750 +2017-09-18 28604730 McKay JD 2017-06-12 Nat Genet www.ncbi.nlm.nih.gov/pubmed/28604730 Large-scale association analysis identifies new lung cancer susceptibility loci and heterogeneity in genetic susceptibility across histological subtypes. Small cell lung carcinoma 2,664 European ancestry cases, 21,444 European ancestry controls NA Illumina [10439017] (imputed) 50 small cell lung carcinoma http://www.ebi.ac.uk/efo/EFO_0000702 GCST004746 Genome-wide genotyping array yes http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004746 diff --git a/tests/gentropy/data_samples/gwas_catalog_studies_sample-r2022-11-29.tsv b/tests/gentropy/data_samples/gwas_catalog_studies_sample-r2022-11-29.tsv deleted file mode 100644 index 7db8f5302..000000000 --- a/tests/gentropy/data_samples/gwas_catalog_studies_sample-r2022-11-29.tsv +++ /dev/null @@ -1,20 +0,0 @@ -DATE ADDED TO CATALOG PUBMED ID FIRST AUTHOR DATE JOURNAL LINK STUDY DISEASE/TRAIT INITIAL SAMPLE SIZE REPLICATION SAMPLE SIZE PLATFORM [SNPS PASSING QC] ASSOCIATION COUNT MAPPED_TRAIT MAPPED_TRAIT_URI STUDY ACCESSION GENOTYPING TECHNOLOGY SUMMARY STATS LOCATION SUBMISSION DATE STATISTICAL MODEL BACKGROUND TRAIT MAPPED BACKGROUND TRAIT MAPPED BACKGROUND TRAIT URI -2019-09-11 26546613 Gutierrez-Achury J 2016-01-01 Hum Mol Genet www.ncbi.nlm.nih.gov/pubmed/26546613 Functional implications of disease-specific variants in loci jointly associated with coeliac disease and rheumatoid arthritis. Celiac disease and Rheumatoid arthritis 371 South Asian ancestry celiac disease cases, 3,138 European ancestry celiac disease cases, 4,418 European ancestry rheumatoid arthritis cases, 509 South Asian ancestry celiac disease controls, 2,473 European ancestry celiac disease controls, 3,300 European ancestry rheumatoid arthritis controls, 8,872 celiac disease cases, 9,401 rheumatoid arthritis cases, 4,845 celiac disease controls, 9,627 rheumatoid arthritis controls NA Illumina [109572] 24 rheumatoid arthritis, celiac disease http://www.ebi.ac.uk/efo/EFO_0000685, http://www.ebi.ac.uk/efo/EFO_0001060 GCST008644 Targeted genotyping array [Immunochip] -2017-05-12 27911795 Schumann G 2016-11-28 Proc Natl Acad Sci U S A www.ncbi.nlm.nih.gov/pubmed/27911795 KLB is associated with alcohol drinking, and its gene product β-Klotho is necessary for FGF21 regulation of alcohol preference. Alcohol consumption up to 70,460 European ancestry drinker individuals up to 35,438 European ancestry drinker individuals Affymetrix, Illumina, Perlegen [at least 316407] (imputed) 3 alcohol consumption measurement http://www.ebi.ac.uk/efo/EFO_0007878 GCST004026 Genome-wide genotyping array -2017-05-12 27911795 Schumann G 2016-11-28 Proc Natl Acad Sci U S A www.ncbi.nlm.nih.gov/pubmed/27911795 KLB is associated with alcohol drinking, and its gene product β-Klotho is necessary for FGF21 regulation of alcohol preference. Alcohol consumption (heavy vs. light/non-drinkers) up to 74,711 European ancestry heavy and light/non-drinker individuals up to 31,021 European ancestry heavy and light/non-drinker individuals Affymetrix, Illumina, Perlegen [at least 316407] (imputed) 2 alcohol consumption measurement http://www.ebi.ac.uk/efo/EFO_0007878 GCST004027 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of organochlorine pesticides 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA Affymetrix [629686] 2 gestational serum measurement, organochlorine pesticide measurement http://www.ebi.ac.uk/efo/EFO_0007964, http://www.ebi.ac.uk/efo/EFO_0007960 GCST004281 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of PCBs (fetal genetic effect) 764 fetuses NA Affymetrix [622716] 11 polychlorinated biphenyls measurement, gestational serum measurement, fetal genotype effect measurement http://www.ebi.ac.uk/efo/EFO_0007042, http://www.ebi.ac.uk/efo/EFO_0007964, http://www.ebi.ac.uk/efo/EFO_0007959 GCST004284 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of PBDEs 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA Affymetrix [629686] 9 polybrominated biphenyl measurement, gestational serum measurement, polybrominated diphenyl ether measurement http://www.ebi.ac.uk/efo/EFO_0007961, http://www.ebi.ac.uk/efo/EFO_0007964, http://www.ebi.ac.uk/efo/EFO_0007962 GCST004285 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of PBDEs (fetal genetic effect) 764 fetuses NA Affymetrix [629686] 11 polybrominated biphenyl measurement, gestational serum measurement, fetal genotype effect measurement, polybrominated diphenyl ether measurement http://www.ebi.ac.uk/efo/EFO_0007961, http://www.ebi.ac.uk/efo/EFO_0007964, http://www.ebi.ac.uk/efo/EFO_0007959, http://www.ebi.ac.uk/efo/EFO_0007962 GCST004286 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of organochlorine pesticides (fetal genetic effect) 764 fetuses NA Affymetrix [629686] 0 gestational serum measurement, fetal genotype effect measurement, organochlorine pesticide measurement http://www.ebi.ac.uk/efo/EFO_0007964, http://www.ebi.ac.uk/efo/EFO_0007959, http://www.ebi.ac.uk/efo/EFO_0007960 GCST004282 Genome-wide genotyping array -2017-07-03 28235828 Traglia M 2017-04-03 G3 (Bethesda) www.ncbi.nlm.nih.gov/pubmed/28235828 Independent Maternal and Fetal Genetic Effects on Midgestational Circulating Levels of Environmental Pollutants. Midgestational circulating levels of PCBs 329 Hispanic mothers, 273 European ancestry mothers, 122 Asian ancestry mothers, 23 South Asian ancestry mothers, 21 African American mothers, 22 other ancestry mothers NA Affymetrix [629686] 25 polychlorinated biphenyls measurement, gestational serum measurement http://www.ebi.ac.uk/efo/EFO_0007042, http://www.ebi.ac.uk/efo/EFO_0007964 GCST004283 Genome-wide genotyping array -2016-09-12 26325155 Brehm JM 2015-09-01 Am J Respir Crit Care Med www.ncbi.nlm.nih.gov/pubmed/26325155 A Genome-Wide Association Study of Post-bronchodilator Lung Function in Children with Asthma. Post-bronchodilator lung function in asthma (FEV1) 447 Puerto Rican ancestry cases 568 European, black or Hispanic cases, 2,414 Hispanic cases Illumina [NR] 0 pulmonary function measurement, forced expiratory volume, response to bronchodilator http://www.ebi.ac.uk/efo/EFO_0003892, http://www.ebi.ac.uk/efo/EFO_0004314, http://purl.obolibrary.org/obo/GO_0097366 GCST003110 Genome-wide genotyping array asthma http://purl.obolibrary.org/obo/MONDO_0004979 -2016-09-12 26325155 Brehm JM 2015-09-01 Am J Respir Crit Care Med www.ncbi.nlm.nih.gov/pubmed/26325155 A Genome-Wide Association Study of Post-bronchodilator Lung Function in Children with Asthma. Post-bronchodilator lung function in asthma (FEV1/FVC) 447 Puerto Rican ancestry cases 568 European, black or Hispanic cases, 2,414 Hispanic cases Illumina [NR] 0 pulmonary function measurement, response to bronchodilator, FEV/FEC ratio http://www.ebi.ac.uk/efo/EFO_0003892, http://purl.obolibrary.org/obo/GO_0097366, http://www.ebi.ac.uk/efo/EFO_0004713 GCST003107 Genome-wide genotyping array asthma http://purl.obolibrary.org/obo/MONDO_0004979 -2016-07-27 26237429 Aminkeng F 2015-08-03 Nat Genet www.ncbi.nlm.nih.gov/pubmed/26237429 A coding variant in RARG confers susceptibility to anthracycline-induced cardiotoxicity in childhood cancer. Anthracycline-induced cardiotoxicity in childhood cancer 32 European ancestry cases with cardiotoxicity, 248 European ancestry cases without cardiotoxicity 22 European ancestry cases with cardiotoxicity, 74 European ancestry cases without cardiotoxicity, 2 African cases with cardiotoxicity, 9 African cases without cardiotoxicity, 5 Hispanic cases with cardiotoxicity, 18 Hispanic cases without cardiotoxicity, 8 East Asian cases with cardiotoxicity, 23 East Asian cases without cardiotoxicity, 4 Aboriginal Canadian cases with cardiotoxicity, 11 Aboriginal Canadian cases without cardiotoxicity Illumina [657694] 2 cardiotoxicity, response to anthracycline-based chemotherapy http://www.ebi.ac.uk/efo/EFO_1001482, http://www.ebi.ac.uk/efo/EFO_0005257 GCST003062 Genome-wide genotyping array childhood cancer http://www.ebi.ac.uk/efo/EFO_1000654 -2017-02-25 27387956 Murk W 2016-07-07 BMC Genet www.ncbi.nlm.nih.gov/pubmed/27387956 Genome-wide search identifies a gene-gene interaction between 20p13 and 2q14 in asthma. Asthma (SNP x SNP interaction) 802 European ancestry cases, 823 European ancestry controls 754 European ancestry cases, 57 European and unknown ancestry cases, 2,573 cases, 880 European ancestry controls, 68 European and unknown ancestry controls, 2,145 controls Illumina [301547] 0 asthma http://purl.obolibrary.org/obo/MONDO_0004979 GCST003682 Genome-wide genotyping array -2016-12-09 27008869 Chen MM 2016-03-23 Hum Mol Genet www.ncbi.nlm.nih.gov/pubmed/27008869 GWAS meta-analysis of 16 852 women identifies new susceptibility locus for endometrial cancer. Endometrial cancer 4,907 European ancestry cases, 11,945 European ancestry controls NA Illumina [9486271] (imputed) 4 endometrial carcinoma http://www.ebi.ac.uk/efo/EFO_1001512 GCST003436 Genome-wide genotyping array -2016-12-01 27016271 Kornilov SA 2016-03-25 Pediatrics www.ncbi.nlm.nih.gov/pubmed/27016271 Genome-Wide Association and Exome Sequencing Study of Language Disorder in an Isolated Population. Developmental language disorder 149 isolated population cases, 210 isolated population controls NA Illumina [223580] 4 specific language impairment http://www.ebi.ac.uk/efo/EFO_1001510 GCST003396 Genome-wide genotyping array -2016-12-01 27016271 Kornilov SA 2016-03-25 Pediatrics www.ncbi.nlm.nih.gov/pubmed/27016271 Genome-Wide Association and Exome Sequencing Study of Language Disorder in an Isolated Population. Developmental language disorder (linguistic errors) 149 isolated population cases, 210 isolated population controls NA Illumina [223580] 9 linguistic error measurement, specific language impairment http://www.ebi.ac.uk/efo/EFO_0007798, http://www.ebi.ac.uk/efo/EFO_1001510 GCST003397 Genome-wide genotyping array -2016-12-01 27016271 Kornilov SA 2016-03-25 Pediatrics www.ncbi.nlm.nih.gov/pubmed/27016271 Genome-Wide Association and Exome Sequencing Study of Language Disorder in an Isolated Population. Developmental language disorder (syntactic complexity) 149 isolated population cases, 210 isolated population controls NA Illumina [223580] 3 syntactic complexity measurement, specific language impairment http://www.ebi.ac.uk/efo/EFO_0007799, http://www.ebi.ac.uk/efo/EFO_1001510 GCST003398 Genome-wide genotyping array -2016-12-13 27005419 Borne Y 2016-03-22 Hum Mol Genet www.ncbi.nlm.nih.gov/pubmed/27005419 Genome wide association study identifies two loci associated with cadmium in erythrocytes among never-smokers. Erythrocyte cadmium concentration 2,704 ever-smoker individuals, 1,728 never-smoker individuals NA Illumina [658884] 5 erythrocyte cadmium measurement http://www.ebi.ac.uk/efo/EFO_0007807 GCST003449 Genome-wide genotyping array -2016-12-13 27005419 Borne Y 2016-03-22 Hum Mol Genet www.ncbi.nlm.nih.gov/pubmed/27005419 Genome wide association study identifies two loci associated with cadmium in erythrocytes among never-smokers. Erythrocyte cadmium concentration in never smokers 1,728 individuals NA Illumina [658884] 7 erythrocyte cadmium measurement http://www.ebi.ac.uk/efo/EFO_0007807 GCST003448 Genome-wide genotyping array diff --git a/utils/update_GWAS_Catalog_data.sh b/utils/update_GWAS_Catalog_data.sh index 1e380d30c..00ee44d8f 100755 --- a/utils/update_GWAS_Catalog_data.sh +++ b/utils/update_GWAS_Catalog_data.sh @@ -15,7 +15,7 @@ get_release_url(){ # Function to get the Ensembl and EFO version which used to ground GWAS data: get_release_info(){ - curl -s https://www.ebi.ac.uk/gwas/api/search/stats | jq -r '"\(.ensemblbuild) \(.efoversion)"' + curl -s "${1}" | jq -r '"\(.ensemblbuild) \(.efoversion)"' } logging(){ @@ -41,6 +41,18 @@ upload_file_to_gcp(){ fi } +fetch_from_ftp(){ + URL=${1} + TARGET=${2} + wget -q ${URL} -O ${TARGET} + if [ $? -ne 0 ]; then + logging "Failed to fetch ${URL}" + return + else + logging "File ${TARGET} saved." + fi +} + # Resources: export BASE_URL=ftp://ftp.ebi.ac.uk/pub/databases/gwas export RELEASE_INFO_URL=https://www.ebi.ac.uk/gwas/api/search/stats @@ -71,7 +83,7 @@ read YEAR MONTH DAY < <(get_release_url) logging "Most recent GWAS Catalog release: ${YEAR}/${MONTH}/${DAY}" # Capturing release metadata: -read ENSEMBL EFO < <(get_release_info) +read ENSEMBL EFO < <(get_release_info ${RELEASE_INFO_URL}) logging "Genes were mapped to v${ENSEMBL} Ensembl release." logging "Diseases were mapped to ${EFO} EFO release." @@ -80,26 +92,19 @@ RELEASE_URL=${BASE_URL}/releases/${YEAR}/${MONTH}/${DAY} logging "Datafiles are fetching from ${RELEASE_URL}" # Fetching files while assigning properly dated and annotated names: -wget -q ${RELEASE_URL}/gwas-catalog-associations_ontology-annotated.tsv -O ${ASSOCIATION_FILE} -logging "File ${ASSOCIATION_FILE} saved." +fetch_from_ftp ${RELEASE_URL}/gwas-catalog-associations_ontology-annotated.tsv ${ASSOCIATION_FILE} -wget -q ${RELEASE_URL}/gwas-catalog-download-studies-v1.0.3.txt -O ${PUBLISHED_STUDIES_FILE} -logging "File ${PUBLISHED_STUDIES_FILE} saved." +fetch_from_ftp ${RELEASE_URL}/gwas-catalog-download-studies-v1.0.3.1.txt ${PUBLISHED_STUDIES_FILE} -wget -q ${RELEASE_URL}/gwas-catalog-unpublished-studies-v1.0.3.tsv -O ${UNPUBLISHED_STUDIES_FILE} -logging "File ${UNPUBLISHED_STUDIES_FILE} saved." +fetch_from_ftp ${RELEASE_URL}/gwas-catalog-unpublished-studies-v1.0.3.1.tsv ${UNPUBLISHED_STUDIES_FILE} -wget -q ${RELEASE_URL}/gwas-catalog-download-ancestries-v1.0.3.txt -O ${PUBLISHED_ANCESTRIES_FILE} -logging "File ${PUBLISHED_ANCESTRIES_FILE} saved." +fetch_from_ftp ${RELEASE_URL}/gwas-catalog-download-ancestries-v1.0.3.1.txt ${PUBLISHED_ANCESTRIES_FILE} -wget -q ${RELEASE_URL}/gwas-catalog-unpublished-ancestries-v1.0.3.tsv -O ${UNPUBLISHED_ANCESTRIES_FILE} -logging "File ${UNPUBLISHED_ANCESTRIES_FILE} saved." +fetch_from_ftp ${RELEASE_URL}/gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv ${UNPUBLISHED_ANCESTRIES_FILE} -wget -q ${BASE_URL}/summary_statistics/harmonised_list.txt -O ${HARMONISED_LIST_FILE} -logging "File ${HARMONISED_LIST_FILE} saved." +fetch_from_ftp ${BASE_URL}/summary_statistics/harmonised_list.txt ${HARMONISED_LIST_FILE} -wget -q ${GWAS_CATALOG_STUDY_CURATION_URL} -O ${GWAS_CATALOG_STUDY_CURATION_FILE} -logging "In-house GWAS Catalog study curation file fetched from GitHub." +fetch_from_ftp ${GWAS_CATALOG_STUDY_CURATION_URL} ${GWAS_CATALOG_STUDY_CURATION_FILE} logging "Copying files to GCP..."