Skip to content

Study Index

gentropy.datasource.eqtl_catalogue.study_index.EqtlCatalogueStudyIndex

Study index dataset from eQTL Catalogue.

We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found here.

One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:

- in the same publication (e.g. Alasoo_2018)
- in the same cell type or tissue (e.g. monocytes)
- and for the same measured molecular trait (e.g. ENSG00000141510)
Source code in src/gentropy/datasource/eqtl_catalogue/study_index.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
class EqtlCatalogueStudyIndex:
    """Study index dataset from eQTL Catalogue.

    We extract study level metadata from eQTL Catalogue's fine mapping results. All available studies can be found [here](https://www.ebi.ac.uk/eqtl/Studies/).

    One study from the eQTL Catalogue clusters together all the molecular QTLs (mQTLs) that were found:

        - in the same publication (e.g. Alasoo_2018)
        - in the same cell type or tissue (e.g. monocytes)
        - and for the same measured molecular trait (e.g. ENSG00000141510)

    """

    raw_studies_metadata_schema: StructType = StructType(
        [
            StructField("study_id", StringType(), True),
            StructField("dataset_id", StringType(), True),
            StructField("study_label", StringType(), True),
            StructField("sample_group", StringType(), True),
            StructField("tissue_id", StringType(), True),
            StructField("tissue_label", StringType(), True),
            StructField("condition_label", StringType(), True),
            StructField("sample_size", IntegerType(), True),
            StructField("quant_method", StringType(), True),
            StructField("pmid", StringType(), True),
            StructField("study_type", StringType(), True),
        ]
    )
    method_to_qtl_type_mapping = {
        QuantificationMethod.GE.value: StudyType.EQTL.value,
        QuantificationMethod.EXON.value: StudyType.EQTL.value,
        QuantificationMethod.TX.value: StudyType.EQTL.value,
        QuantificationMethod.MICROARRAY.value: StudyType.EQTL.value,
        QuantificationMethod.LEAFCUTTER.value: StudyType.SQTL.value,
        QuantificationMethod.APTAMER.value: StudyType.PQTL.value,
        QuantificationMethod.TXREV.value: StudyType.TUQTL.value,
        QuantificationMethod.MAJIQ.value: StudyType.SQTL.value,
    }

    @classmethod
    def _identify_study_type(
        cls: type[EqtlCatalogueStudyIndex],
    ) -> Column:
        """Identify the qtl type based on the quantification method and eqtl catalogue study type.

        Returns:
            Column: The study type.

        Examples:
            >>> df = spark.createDataFrame([("ge", "bulk"), ("leafcutter", "bulk"), ("tx", "single-cell")], ["quant_method", "study_type"])
            >>> df.withColumn("studyType", EqtlCatalogueStudyIndex._identify_study_type()).show()
            +------------+-----------+---------+
            |quant_method| study_type|studyType|
            +------------+-----------+---------+
            |          ge|       bulk|     eqtl|
            |  leafcutter|       bulk|     sqtl|
            |          tx|single-cell|   sceqtl|
            +------------+-----------+---------+
            <BLANKLINE>
        """
        qtl_type_mapping = f.create_map(
            *[f.lit(x) for x in chain(*cls.method_to_qtl_type_mapping.items())]
        )[f.col("quant_method")]
        return f.when(
            f.col("study_type") == "single-cell",
            f.concat(f.lit("sc"), qtl_type_mapping),
        ).otherwise(qtl_type_mapping)

    @classmethod
    def from_susie_results(
        cls: type[EqtlCatalogueStudyIndex],
        processed_finemapping_df: DataFrame,
    ) -> StudyIndex:
        """Ingest study level metadata from eQTL Catalogue.

        Args:
            processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.

        Returns:
            StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.
        """
        study_index_cols = [
            field.name
            for field in StudyIndex.get_schema().fields
            if field.name in processed_finemapping_df.columns
        ]
        return StudyIndex(
            _df=processed_finemapping_df.select(study_index_cols).distinct(),
            _schema=StudyIndex.get_schema(),
        )

    @classmethod
    def read_studies_from_source(
        cls: type[EqtlCatalogueStudyIndex],
        metadata_path: str,
        mqtl_quantification_methods_blacklist: list[str],
        session: Session | None = None,
    ) -> DataFrame:
        """Read raw studies metadata from eQTL Catalogue.

        Args:
            metadata_path (str): Path to the studies metadata file.
            mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv
            session (Session | None, optional): Session object. If not provided, the method will try to find an active session. Defaults to None.

        Returns:
            DataFrame: raw studies metadata.

        Raises:
            ValueError: If an invalid quantification method is provided in the blacklist.

        Example metadata_path: "https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/fe3c4b4ed911b3a184271a6aadcd8c8769a66aba/data_tables/dataset_metadata.tsv"
        """
        for method in mqtl_quantification_methods_blacklist:
            if method not in cls.method_to_qtl_type_mapping:
                raise ValueError(
                    f"Quantification method '{method}' is not supported. "
                    + f"Available options are: {list(cls.method_to_qtl_type_mapping.keys())}"
                )
        session = session or Session.find()
        return session.load_data(
            metadata_path,
            schema=cls.raw_studies_metadata_schema,
            fmt=NativeFileFormat.TSV.value,
        ).filter(~(f.col("quant_method").isin(mqtl_quantification_methods_blacklist)))

from_susie_results(processed_finemapping_df: DataFrame) -> StudyIndex classmethod

Ingest study level metadata from eQTL Catalogue.

Parameters:

Name Type Description Default
processed_finemapping_df DataFrame

processed fine mapping results with study metadata.

 required

Returns:

Name Type Description
StudyIndex StudyIndex

eQTL Catalogue study index dataset derived from the selected SuSIE results.
Source code in src/gentropy/datasource/eqtl_catalogue/study_index.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@classmethod
def from_susie_results(
    cls: type[EqtlCatalogueStudyIndex],
    processed_finemapping_df: DataFrame,
) -> StudyIndex:
    """Ingest study level metadata from eQTL Catalogue.

    Args:
        processed_finemapping_df (DataFrame): processed fine mapping results with study metadata.

    Returns:
        StudyIndex: eQTL Catalogue study index dataset derived from the selected SuSIE results.
    """
    study_index_cols = [
        field.name
        for field in StudyIndex.get_schema().fields
        if field.name in processed_finemapping_df.columns
    ]
    return StudyIndex(
        _df=processed_finemapping_df.select(study_index_cols).distinct(),
        _schema=StudyIndex.get_schema(),
    )

read_studies_from_source(metadata_path: str, mqtl_quantification_methods_blacklist: list[str], session: Session | None = None) -> DataFrame classmethod

Read raw studies metadata from eQTL Catalogue.

Parameters:

Name Type Description Default
metadata_path str

Path to the studies metadata file.

 required
mqtl_quantification_methods_blacklist list[str]

Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv

 required
session Session | None

Session object. If not provided, the method will try to find an active session. Defaults to None.

 None

Returns:

Name Type Description
DataFrame DataFrame

raw studies metadata.

Raises:

Type Description
ValueError

If an invalid quantification method is provided in the blacklist.

Example metadata_path: "https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/fe3c4b4ed911b3a184271a6aadcd8c8769a66aba/data_tables/dataset_metadata.tsv"

Source code in src/gentropy/datasource/eqtl_catalogue/study_index.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
@classmethod
def read_studies_from_source(
    cls: type[EqtlCatalogueStudyIndex],
    metadata_path: str,
    mqtl_quantification_methods_blacklist: list[str],
    session: Session | None = None,
) -> DataFrame:
    """Read raw studies metadata from eQTL Catalogue.

    Args:
        metadata_path (str): Path to the studies metadata file.
        mqtl_quantification_methods_blacklist (list[str]): Molecular trait quantification methods that we don't want to ingest. Available options in https://github.com/eQTL-Catalogue/eQTL-Catalogue-resources/blob/master/data_tables/dataset_metadata.tsv
        session (Session | None, optional): Session object. If not provided, the method will try to find an active session. Defaults to None.

    Returns:
        DataFrame: raw studies metadata.

    Raises:
        ValueError: If an invalid quantification method is provided in the blacklist.

    Example metadata_path: "https://raw.githubusercontent.com/eQTL-Catalogue/eQTL-Catalogue-resources/fe3c4b4ed911b3a184271a6aadcd8c8769a66aba/data_tables/dataset_metadata.tsv"
    """
    for method in mqtl_quantification_methods_blacklist:
        if method not in cls.method_to_qtl_type_mapping:
            raise ValueError(
                f"Quantification method '{method}' is not supported. "
                + f"Available options are: {list(cls.method_to_qtl_type_mapping.keys())}"
            )
    session = session or Session.find()
    return session.load_data(
        metadata_path,
        schema=cls.raw_studies_metadata_schema,
        fmt=NativeFileFormat.TSV.value,
    ).filter(~(f.col("quant_method").isin(mqtl_quantification_methods_blacklist)))
  • 2023-11-05
  • 2024-01-17
  • Contributors
    • d0choa
    • web-flow