Ddp

artimes_experiments/iuxray_rgmg/config.yml

@@ -0,0 +1,144 @@
+LOGGER_NAME: 'log'
+SEED: 1546884941.160048
+
+############################ TRAIN ############################
+TRAIN:
+  BATCH_SIZE: 16
+  #################### REINFORCEMENT ####################
+  REINFORCEMENT:
+    START: 9999
+
+############################ TEST ############################
+TEST:
+  BATCH_SIZE: 16
+
+############################ DATA_LOADER ############################
+DATA_LOADER:
+  NUM_WORKERS: 4
+  SHUFFLE: True
+  TRAIN_GV_FEAT: ''
+  TRAIN_ATT_FEATS: './mscoco/feature/up_down_100'
+  VAL_GV_FEAT: ''
+  VAL_ATT_FEATS: './mscoco/feature/up_down_100'
+  TEST_GV_FEAT: ''
+  TEST_ATT_FEATS: './mscoco/feature/up_down_100'
+  TRAIN_ID: './mscoco/txt/coco_train_image_id.txt'
+  VAL_ID: './mscoco/txt/coco_val_image_id.txt'
+  TEST_ID: './mscoco/txt/coco_test_image_id.txt'
+  INPUT_SEQ_PATH: './mscoco/sent/coco_train_input.pkl'
+  TARGET_SEQ_PATH: './mscoco/sent/coco_train_target.pkl'
+  SEQ_PER_IMG: 1
+  MAX_FEAT: 50
+
+############################ MODEL ############################
+MODEL:
+  TYPE: 'XTransformer'
+  SEQ_LEN: 60   # Modified                               # include <EOS>/<BOS>
+  VOCAB_SIZE: 760   # TODO                       # exclude <EOS>/<BOS> IUXRAY: 760
+  ########## word embedding ##########
+  WORD_EMBED_DIM: 768
+  WORD_EMBED_ACT: 'CELU'
+  WORD_EMBED_NORM: False
+  DROPOUT_WORD_EMBED: 0.1
+  ########## global features ##########
+  GVFEAT_DIM: 2048
+  GVFEAT_EMBED_DIM: -1
+  GVFEAT_EMBED_ACT: 'NONE'
+  DROPOUT_GV_EMBED: 0.0
+  ########## attention features ##########
+  ATT_FEATS_DIM: 1024   # Modified
+  ATT_FEATS_EMBED_DIM: 768
+  ATT_FEATS_EMBED_ACT: 'CELU'
+  DROPOUT_ATT_EMBED: 0.5
+  ATT_FEATS_NORM: True
+  ########## attention param ##########
+  ATT_HIDDEN_SIZE: -1
+  ATT_HIDDEN_DROP: 0.0
+  ATT_ACT: 'TANH'
+  ########## rnn param ##########
+  RNN_SIZE: 1024
+  DROPOUT_LM: 0.5
+
+  ########## BOTTOM_UP ##########
+  BOTTOM_UP:
+    DROPOUT_FIRST_INPUT: 0.0
+    DROPOUT_SEC_INPUT: 0.0
+
+  ########## BILINEAR ##########
+  BILINEAR:
+    DIM: 768
+    ENCODE_ATT_MID_DIM: [96, 48, 96]
+    DECODE_ATT_MID_DIM: [96, 48, 96]
+    ENCODE_ATT_MID_DROPOUT: 0.1
+    DECODE_ATT_MID_DROPOUT: 0.1
+    ATT_DIM: 768
+    ACT: 'CELU'
+    ENCODE_DROPOUT: 0.5
+    DECODE_DROPOUT: 0.5
+    ENCODE_LAYERS: 6
+    DECODE_LAYERS: 6
+    TYPE: 'LowRank'
+    ATTTYPE: 'SCAtt' # SCAtt, BasicAtt
+    HEAD: 8
+    ENCODE_FF_DROPOUT: 0.5
+    DECODE_FF_DROPOUT: 0.5
+    ELU_ALPHA: 1.3
+    BIFEAT_EMB_ACT: 'RELU'
+    ENCODE_BIFEAT_EMB_DROPOUT: 0.3
+    DECODE_BIFEAT_EMB_DROPOUT: 0.3
+
+############################ SOLVER ############################
+SOLVER:
+  BASE_LR: 0.000001
+  TYPE: 'RADAM'                 # 'ADAM', 'SGD', 'RADAM'
+  MAX_EPOCH: 20
+  MAX_ITER: -1
+  GRAD_CLIP: 0.1                # Norm:0.5 , Clamp:0.1
+  GRAD_CLIP_TYPE: 'Clamp'       # 'Clamp'  , 'Norm'
+  WEIGHT_DECAY: 0.0000
+  WEIGHT_DECAY_BIAS: 0.0
+  BIAS_LR_FACTOR: 1
+  DISPLAY: 100
+  TEST_INTERVAL: 1
+  SNAPSHOT_ITERS: 1
+
+  ########## SGD ##########
+  SGD:
+    MOMENTUM: 0.9
+  ########## ADAM ##########
+  ADAM:
+    BETAS: [0.9, 0.98]
+    EPS: 1.0e-9
+  ########## LR_POLICY ##########
+  LR_POLICY:
+    TYPE: 'Noam'              # 'Fix', 'Step', 'Noam', 'Plateau'
+    GAMMA: 0.8
+    STEP_SIZE: 3
+    SETP_TYPE: 'Iter'         # 'Epoch', 'Iter'
+    WARMUP: 10000             #  For Noam only
+    FACTOR: 1.0               #  For Noam only
+    MODEL_SIZE: 768           #  For Noam only
+
+############################ LOSSES ############################
+LOSSES:
+  XE_TYPE: 'LabelSmoothing'     # 'CrossEntropy', 'LabelSmoothing'
+  LABELSMOOTHING: 0.1
+  RL_TYPE: 'RewardCriterion'
+
+############################ SCORER ############################
+SCORER:
+  TYPES: ['CIDEr']
+  WEIGHTS: [1.0]
+  GT_PATH: './mscoco/misc/coco_train_gts.pkl'
+  CIDER_CACHED: './mscoco/misc/coco_train_cider.pkl'
+
+############################ INFERENCE ############################
+INFERENCE:
+  VOCAB: './mscoco/txt/coco_vocabulary.txt'
+  ID_KEY: 'image_id'
+  CAP_KEY: 'caption'
+  EVAL: 'COCO'
+  VAL_ANNFILE: './mscoco/misc/captions_val5k.json'
+  TEST_ANNFILE: './mscoco/misc/captions_test5k.json'
+  BEAM_SIZE: 2
+  GREEDY_DECODE: True

artimes_experiments/iuxray_rgmg/train.sh

@@ -0,0 +1,7 @@
+#CUDA_VISIBLE_DEVICES=0  -m torch.distributed.launch --nproc_per_node=1
+python3 main.py --folder ./artimes_experiments/iuxray_rgmg  --resume 0 --submodel rgmg --KG_path /project/CVML/pretrained_kg/rgmg_iuxray_pretrain.pth --dataset_name IUXRAY --image_dir /project/CVML/Parallel-R2Gen-KG/data/iu_xray/images/ --ann_path /project/CVML/Parallel-R2Gen-KG/data/iuxray/annotation.json
+
+
+
+
+###if you want to use checkpoint, download your model in experiments_mimiccxr/xtransformer/snapshot and change 0 to your model's number

datasets/__init__.py

@@ -0,0 +1,19 @@
+from datasets.coco_dataset import CocoDataset
+from datasets.radiology_dataset import  IUXRAY
+from datasets.radiology_dataset import MIMICCXR
+from datasets.radiology_dataset import MimiccxrMultiImage
+
+__factory = {
+    'IUXRAY': IUXRAY,
+    'MIMICCXR': MIMICCXR,
+    'MIMICCXR_MultiImages': MimiccxrMultiImage,
+    'COCO': CocoDataset,
+}
+
+def names():
+    return sorted(__factory.keys())
+
+def create(name, *args, **kwargs):
+    if name not in __factory:
+        raise KeyError("Unknown Dataset:", name)
+    return __factory[name](*args, **kwargs)