mimic3 benchmark代码解析（一）

git clone https://github.com/YerevaNN/mimic3-benchmarks/
cd mimic3-benchmarks/

环境

numpy == 1.13.1，pandas == 0.20.3

mimic3-benchmarks-master\mimic3benchmark\scripts\extract_subjects

为每个SUBJECT_ID生成一个文件，将ICU住院信息写入data/{SUBJECT_ID}/stays.csv，诊断写入data/{SUBJECT_ID}/diagnoses.csv，事件写入data/{SUBJECT_ID}/events.csv，这大约需要一个小时。
执行命令：

python -m mimic3benchmark.scripts.extract_subjects {PATH TO MIMIC-III CSVs} data/root/

extract_subjects.py

from __future__ import absolute_import
from __future__ import print_function

import argparse
import yaml
import os

from mimic3benchmark.mimic3csv import *
from mimic3benchmark.preprocessing import add_hcup_ccs_2015_groups, make_phenotype_label_matrix
from mimic3benchmark.util import *
#参数设置
parser = argparse.ArgumentParser(description='Extract per-subject data from MIMIC-III CSV files.')
parser.add_argument('mimic3_path', type=str, help='Directory containing MIMIC-III CSV files.')
parser.add_argument('output_path', type=str, help='Directory where per-subject data should be written.')
parser.add_argument('--event_tables', '-e', type=str, nargs='+', help='Tables from which to read events.',
                    default=['CHARTEVENTS', 'LABEVENTS', 'OUTPUTEVENTS'])
parser.add_argument('--phenotype_definitions', '-p', type=str,
                    default=os.path.join(os.path.dirname(__file__), '../resources/hcup_ccs_2015_definitions.yaml'),
                    help='YAML file with phenotype definitions.')
parser.add_argument('--itemids_file', '-i', type=str, help='CSV containing list of ITEMIDs to keep.')
parser.add_argument('--verbose', '-v', type=int, help='Level of verbosity in output.', default=1)
parser.add_argument('--test', action='store_true', help='TEST MODE: process only 1000 subjects, 1000000 events.')
args, _ = parser.parse_known_args()
#建立输出文件路径
try:
    os.makedirs(args.output_path)
except:
    pass
#读取解压开的.csv文件
patients = read_patients_table(args.mimic3_path) #读取PATIENTS.csv
admits = read_admissions_table(args.mimic3_path) #读取ADMISSIONS.csv
stays = read_icustays_table(args.mimic3_path) #读取ICUSTAYS.csv
if args.verbose: #True
    print('START:', stays.ICUSTAY_ID.unique().shape[0], stays.HADM_ID.unique().shape[0],
          stays.SUBJECT_ID.unique().shape[0])

stays = remove_icustays_with_transfers(stays)  #筛选
if args.verbose:
    print('REMOVE ICU TRANSFERS:', stays.ICUSTAY_ID.unique().shape[0], stays.HADM_ID.unique().shape[0],
          stays.SUBJECT_ID.unique().shape[0])

stays = merge_on_subject_admission(stays, admits)  
#在表stays, admit中把'SUBJECT_ID', 'HADM_ID'相等的行合并成新的表
stays = merge_on_subject(stays, patients)  
#在表stays, patients中把'SUBJECT_ID'相等的行合并成新的表
stays = filter_admissions_on_nb_icustays(stays)
#选择stays中HADM_ID只出现一次的行
if args.verbose:
    print('REMOVE MULTIPLE STAYS PER ADMIT:', stays.ICUSTAY_ID.unique().shape[0], stays.HADM_ID.unique().shape[0],
          stays.SUBJECT_ID.unique().shape[0])

stays = add_age_to_icustays(stays) #计算年龄，增加年龄列
stays = add_inunit_mortality_to_icustays(stays) #添加是否在ICU死亡列
stays = add_inhospital_mortality_to_icustays(stays)#添加是否在医院死亡列
stays = filter_icustays_on_age(stays) #年龄大于等于18岁的行
if args.verbose:
    print('REMOVE PATIENTS AGE < 18:', stays.ICUSTAY_ID.unique().shape[0], stays.HADM_ID.unique().shape[0],
          stays.SUBJECT_ID.unique().shape[0])

stays.to_csv(os.path.join(args.output_path, 'all_stays.csv'), index=False)
diagnoses = read_icd_diagnoses_table(args.mimic3_path) 
#读取D_ICD_DIAGNOSES.csv和DIAGNOSES_ICD.csv，再通过ICD9_CODE拼接
diagnoses = filter_diagnoses_on_stays(diagnoses, stays) 
# 将stays的'SUBJECT_ID', 'HADM_ID', 'ICUSTAY_ID'和diagnoses按'SUBJECT_ID', 'HADM_ID'拼接
diagnoses.to_csv(os.path.join(args.output_path, 'all_diagnoses.csv'), index=False)
count_icd_codes(diagnoses, output_path=os.path.join(args.output_path, 'diagnosis_counts.csv'))
#diagnoses的'ICD9_CODE'（会作为索引，和分组）, 'SHORT_TITLE', 'LONG_TITLE'列，增加COUNT列
phenotypes = add_hcup_ccs_2015_groups(diagnoses, yaml.load(open(args.phenotype_definitions, 'r')))
#diagnoses增加两列'HCUP_CCS_2015'，'USE_IN_BENCHMARK'
make_phenotype_label_matrix(phenotypes, stays).to_csv(os.path.join(args.output_path, 'phenotype_labels.csv'),
                                                      index=False, quoting=csv.QUOTE_NONNUMERIC)

if args.test:
    pat_idx = np.random.choice(patients.shape[0], size=1000)
    patients = patients.iloc[pat_idx]
    stays = stays.merge(patients[['SUBJECT_ID']], left_on='SUBJECT_ID', right_on='SUBJECT_ID')
    args.event_tables = [args.event_tables[0]]
    print('Using only', stays.shape[0], 'stays and only', args.event_tables[0], 'table')

subjects = stays.SUBJECT_ID.unique()
break_up_stays_by_subject(stays, args.output_path, subjects=subjects, verbose=args.verbose)
break_up_diagnoses_by_subject(phenotypes, args.output_path, subjects=subjects, verbose=args.verbose)
items_to_keep = set(
    [int(itemid) for itemid in dataframe_from_csv(args.itemids_file)['ITEMID'].unique()]) if args.itemids_file else None
for table in args.event_tables:
    read_events_table_and_break_up_by_subject(args.mimic3_path, table, args.output_path, items_to_keep=items_to_keep,
                                              subjects_to_keep=subjects, verbose=args.verbose)

mimic3csv.py

from __future__ import absolute_import
from __future__ import print_function

import csv
import numpy as np
import os
import pandas as pd
import sys

from mimic3benchmark.util import *

#读取PATIENTS.csv
def read_patients_table(mimic3_path):
    pats = dataframe_from_csv(os.path.join(mimic3_path, 'PATIENTS.csv'))
    pats = pats[['SUBJECT_ID', 'GENDER', 'DOB', 'DOD']] #取这4列
    pats.DOB = pd.to_datetime(pats.DOB) #将DOB列数据转换为日期时间
    pats.DOD = pd.to_datetime(pats.DOD)
    return pats

#读取ADMISSIONS.csv
def read_admissions_table(mimic3_path):
    admits = dataframe_from_csv(os.path.join(mimic3_path, 'ADMISSIONS.csv'))
    admits = admits[['SUBJECT_ID', 'HADM_ID', 'ADMITTIME', 'DISCHTIME', 'DEATHTIME', 'ETHNICITY', 'DIAGNOSIS']] #取这7列
    admits.ADMITTIME = pd.to_datetime(admits.ADMITTIME)
    admits.DISCHTIME = pd.to_datetime(admits.DISCHTIME)
    admits.DEATHTIME = pd.to_datetime(admits.DEATHTIME)
    return admits


def read_icustays_table(mimic3_path):
    stays = dataframe_from_csv(os.path.join(mimic3_path, 'ICUSTAYS.csv'))
    stays.INTIME = pd.to_datetime(stays.INTIME)
    stays.OUTTIME = pd.to_datetime(stays.OUTTIME)
    return stays


def read_icd_diagnoses_table(mimic3_path):
    codes = dataframe_from_csv(os.path.join(mimic3_path, 'D_ICD_DIAGNOSES.csv'))
    codes = codes[['ICD9_CODE', 'SHORT_TITLE', 'LONG_TITLE']]
    diagnoses = dataframe_from_csv(os.path.join(mimic3_path, 'DIAGNOSES_ICD.csv'))
    diagnoses = diagnoses.merge(codes, how='inner', left_on='ICD9_CODE', right_on='ICD9_CODE')
    diagnoses[['SUBJECT_ID', 'HADM_ID', 'SEQ_NUM']] = diagnoses[['SUBJECT_ID', 'HADM_ID', 'SEQ_NUM']].astype(int)
    return diagnoses


def read_events_table_by_row(mimic3_path, table):
    nb_rows = {
   
   'chartevents': 330712484, 'labevents': 27854056, 'outputevents': 4349219}
    reader = csv.DictReader(open(os.path.join(mimic3_path, table.upper() + '.csv'), 'r'))
    for i, row in enumerate(reader):
        if 'ICUSTAY_ID' not in row:
            row['ICUSTAY_ID'] = ''
        yield row, i, nb_rows[table.lower()]


def count_icd_codes(diagnoses, output_path=None):
    codes = diagnoses[['ICD9_CODE', 'SHORT_TITLE', 'LONG_TITLE']].drop_duplicates().set_index('ICD9_CODE')
    #取diagnoses的这3列并除去重复列，再将ICD9_CODE设置为索引
    codes['COUNT'] = diagnoses.groupby('ICD9_CODE')['ICUSTAY_ID'].count()
    #按ICD9_CODE分组，并计算每组的个数
    codes.COUNT = codes.COUNT.fillna