From cbcda43c9609c207004ee77a51aa4102422e0bb9 Mon Sep 17 00:00:00 2001 From: vardaan123 Date: Sun, 7 Jan 2024 20:49:37 +0000 Subject: [PATCH] remove files --- README.md | 106 ---- dataset.py | 147 ----- eval.py | 189 ------ gen_utils/analyze_img_loc.py | 219 ------- gen_utils/analyze_img_time.py | 127 ----- gen_utils/analyze_taxonomy_model.py | 362 ------------ gen_utils/dump_imageonly_pred_specie_wise.py | 236 -------- gen_utils/dump_kge_pred_specie_wise.py | 211 ------- gen_utils/eval_kge_specie_wise.py | 102 ---- main.py | 570 ------------------- model.py | 191 ------- preprocess_data_iwildcam.py | 207 ------- preprocess_data_mountain_zebra.py | 227 -------- preprocess_iwildcam.sh | 7 - preprocess_mountain_zebra.sh | 11 - process_taxonomy_inat.py | 103 ---- process_taxonomy_ott.py | 21 - pytorchtools.py | 84 --- requirements.txt | 10 - resnet.py | 33 -- run_image_only_model.py | 269 --------- utils.py | 74 --- 22 files changed, 3506 deletions(-) delete mode 100644 README.md delete mode 100644 dataset.py delete mode 100644 eval.py delete mode 100644 gen_utils/analyze_img_loc.py delete mode 100644 gen_utils/analyze_img_time.py delete mode 100644 gen_utils/analyze_taxonomy_model.py delete mode 100644 gen_utils/dump_imageonly_pred_specie_wise.py delete mode 100644 gen_utils/dump_kge_pred_specie_wise.py delete mode 100644 gen_utils/eval_kge_specie_wise.py delete mode 100644 main.py delete mode 100644 model.py delete mode 100644 preprocess_data_iwildcam.py delete mode 100644 preprocess_data_mountain_zebra.py delete mode 100644 preprocess_iwildcam.sh delete mode 100644 preprocess_mountain_zebra.sh delete mode 100644 process_taxonomy_inat.py delete mode 100644 process_taxonomy_ott.py delete mode 100644 pytorchtools.py delete mode 100644 requirements.txt delete mode 100644 resnet.py delete mode 100644 run_image_only_model.py delete mode 100644 utils.py diff --git a/README.md b/README.md deleted file mode 100644 index d665ce6..0000000 --- a/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# Installation - -``` -pip install -r requirements.txt -``` - -# Data Preprocessing - -## iWildCam2020-WILDS -``` -bash preprocess_iwildcam.sh -``` -Note: The dir. `data/iwildcam_v2.0/train/` contains images for all splits. - -## Snapshot Mountain Zebra -1. Download snapshot_mountain_zebra.zip from [this link](https://buckeyemailosu-my.sharepoint.com/:u:/g/personal/pahuja_9_buckeyemail_osu_edu/EWI05mXQsopNskBo78a_l_ABSZJHl0uCsdNMu72aXmNNiA?e=LOtm5Q) and uncompress it into a directory `data/snapshot_mountain_zebra/`. -2. Download images using the command `gsutil -m cp -r "gs://public-datasets-lila/snapshot-safari/MTZ/MTZ_public" data/snapshot_mountain_zebra/` -2. Run `bash preprocess_mountain_zebra.sh` - - -# Training - -Note: The below commands will use the DistMult model by default. Use the following hyperparameter configuration: - -- For iWildCam2020-WILDS, set `DATA_DIR` to `data/iwildcam_v2.0/`, `IMG_DIR` to `data/iwildcam_v2.0/train/`, and `DATASET` to `iwildcam` -- For Snapshot Mountain Zebra, set `DATA_DIR` to `data/snapshot_mountain_zebra/` and `IMG_DIR` to `data/snapshot_mountain_zebra/`, and `DATASET` to `mountain_zebra`. -- For ConvE, use `--kg-embed-model conve --embedding-dim 200` in args. - - -## Image-only model (ERM baseline) -``` -python -u run_image_only_model.py --data-dir DATA_DIR --img-dir IMG_DIR --save-dir CKPT_DIR > CKPT_DIR/log.txt -``` - -## COSMO, no-context baseline -``` -python -u main.py --data-dir DATA_DIR --img-dir IMG_DIR --save-dir CKPT_DIR > CKPT_DIR/log.txt -``` - -## COSMO, taxonomy -``` -python -u main.py --data-dir DATA_DIR --img-dir IMG_DIR --save-dir CKPT_DIR --add-id-id > CKPT_DIR/log.txt -``` - -## COSMO, location -``` -python -u main.py --data-dir DATA_DIR --img-dir IMG_DIR --save-dir CKPT_DIR --add-image-location > CKPT_DIR/log.txt -``` - -## COSMO, time -``` -python -u main.py --data-dir DATA_DIR --img-dir IMG_DIR/ --save-dir CKPT_DIR --add-image-time > CKPT_DIR/log.txt -``` - -## COSMO, taxonomy + location + time -``` -python -u main.py --data-dir DATA_DIR --img-dir IMG_DIR --save-dir CKPT_DIR --add-id-id --add-image-time --add-image-location > CKPT_DIR/log.txt -``` - -# Evaluation - -## Evaluate a model (specify split) -``` -python eval.py --ckpt-path --split test --data-dir DATA_DIR --img-dir IMG_DIR -``` - -# Error Analysis - -## Taxonomy analysis -``` -cd gen_utils/ -python analyze_taxonomy_model.py --data-dir DATA_DIR --img-dir IMG_DIR --ckpt-1-path --ckpt-2-path -``` - -## Plot location correlation analysis -``` -cd gen_utils/ -python analyze_img_loc.py --data-dir DATA_DIR -``` - -## Plot time correlation analysis -``` -cd gen_utils/ -python analyze_img_time.py --data-dir DATA_DIR -``` - - -## Under-represented Species Analysis - -### Dump predictions for baseline image-only model -``` -cd gen_utils/ -python dump_imageonly_pred_specie_wise.py --ckpt-path --split test --out-dir -``` - -### Dump predictions for COSMO model -``` -cd gen_utils/ -python dump_kge_pred_specie_wise.py --ckpt-path --split test --out-dir -``` - -### Compare performance for under-represented species -``` -cd gen_utils/ -python eval_kge_specie_wise.py --y-pred-path-1 --y-pred-path-2 -``` diff --git a/dataset.py b/dataset.py deleted file mode 100644 index f99e4d3..0000000 --- a/dataset.py +++ /dev/null @@ -1,147 +0,0 @@ -import os -import torch -import numpy as np -import re -from math import pi -from re import match -from PIL import Image -from torchvision import transforms -from torch.utils.data import Dataset, DataLoader -import json -import pickle -import pandas as pd - -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN = [0.485, 0.456, 0.406] -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD = [0.229, 0.224, 0.225] - -class iWildCamOTTDataset(Dataset): - def __init__(self, datacsv, mode, args, entity2id, target_list, head_type=None, tail_type=None): - super(iWildCamOTTDataset, self).__init__() - if head_type is not None and tail_type is not None: - self.datacsv = datacsv.loc[(datacsv['datatype_h'] == head_type) & (datacsv['datatype_t'] == tail_type) & ( - datacsv['split'] == mode), :] - print("length of {}2{} dataset = {}".format(head_type, tail_type, len(self.datacsv))) - else: - self.datacsv = datacsv.loc[datacsv['split'] == mode, :] - print("length of alltype dataset = {}".format(len(self.datacsv))) - self.args = args - self.mode = mode - self.entity2id = entity2id - self.target_list = target_list - self.entity_to_species_id = {self.target_list[i, 0].item():i for i in range(len(self.target_list))} - - # print(self.entity_to_species_id) - - if args.use_data_subset: - train_indices = np.random.choice(np.arange(len(self.datacsv)), size=args.subset_size, replace=False) - self.datacsv = self.datacsv.iloc[train_indices] - - if head_type == 'image' and tail_type == 'location': - datacsv_loc = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'location')] - - self.location_to_id = {} - - for i in range(len(datacsv_loc)): - loc = datacsv_loc.iloc[i, 3] - - assert loc[0] == '[' - assert loc[-1] == ']' - # print(loc) - if loc not in self.location_to_id: - self.location_to_id[loc] = len(self.location_to_id) - - self.all_locs = torch.stack(list(map(lambda x:getNumber(x), self.location_to_id.keys()))) - - self.all_timestamps = None - - if head_type == 'image' and tail_type == 'time': - datacsv_time = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'time')] - self.time_to_id = {} - - for i in range(len(datacsv_time)): - time = datacsv_time.iloc[i, 3] - - _, hour = get_separate_time(time) - - _HOUR_RAD = 2 * pi / 24 - - h1, h2 = point(hour, _HOUR_RAD) - - time = hour - - if time not in self.time_to_id: - self.time_to_id[time] = len(self.time_to_id) - - self.all_timestamps = torch.stack(list(map(lambda x:torch.tensor(x), self.time_to_id.keys()))) - if len(self.all_timestamps.size())==1: - self.all_timestamps = self.all_timestamps.unsqueeze(-1) - - def __len__(self): - return len(self.datacsv) - - def __getitem__(self, idx): - - head_type = self.datacsv.iloc[idx, 1] - tail_type = self.datacsv.iloc[idx, 4] - head = self.datacsv.iloc[idx, 0] - relation = self.datacsv.iloc[idx, 2] - tail = self.datacsv.iloc[idx, 3] - - # for tail extract - h = None - t = None - - if tail_type == "id": - if head_type in ["image", "location"]: - t = torch.tensor([self.entity_to_species_id[self.entity2id[str(int(float(tail)))]]], dtype=torch.long).squeeze(-1) - else: - t = torch.tensor([self.entity2id[str(int(float(tail)))]], dtype=torch.long).squeeze(-1) - - elif tail_type == "location": - t = self.location_to_id[tail] - - elif tail_type == "time": - tail = datatime_divide(tail, self.args) - t = self.time_to_id[tail] - - # for head extract - if head_type == "id": - h = torch.tensor([self.entity2id[str(int(float(head)))]], dtype=torch.long).squeeze(-1) - - elif head_type == "image": - img = Image.open(os.path.join(self.args.img_dir, head)).convert('RGB') - - transform_steps = transforms.Compose([transforms.Resize((448, 448)), transforms.ToTensor(), transforms.Normalize(_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN, _DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD)]) - h = transform_steps(img) - - elif head_type == "location": - h = getNumber(head) - - # for r extract - r = torch.tensor([int(relation)]) - - return h, r, t - -def getNumber(x): - return torch.tensor(np.fromstring(x[1:-1], dtype=float, sep=' '), dtype=torch.float) - -def get_separate_time(item): - m = match(r"(.*)-(.*)-(.*) (.*):(.*):(\d{2})", item) - years, month, day, hour, minutes, second = m.groups() - return float(month), float(hour) - -def datatime_divide(timestamp, args): - month, hour = get_separate_time(timestamp) - - _HOUR_RAD = 2 * pi / 24 - - h1, h2 = point(hour, _HOUR_RAD) - - return hour - -def point(m, rad): - from math import sin, cos - # place on circle - return sin(m * rad), cos(m * rad) - - diff --git a/eval.py b/eval.py deleted file mode 100644 index 1d65a58..0000000 --- a/eval.py +++ /dev/null @@ -1,189 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict -import math - -sys.path.append('../') - -from model import MKGE -from resnet import Resnet18, Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy -from PIL import Image -from dataset import iWildCamOTTDataset - -def evaluate(model, val_loader, target_list, args): - model.eval() - torch.set_grad_enabled(False) - - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - for labeled_batch in tqdm(val_loader): - h, r, t = labeled_batch - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'id')) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - y_true = detach_and_clone(batch_results['y_true']) - epoch_y_true.append(y_true) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - batch_idx += 1 - if args.debug: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - print(f'Eval., split: {args.split}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100:.2f}') - - return - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - # print('categories = {}'.format(categories)) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - - - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--dataset', choices=['iwildcam', 'mountain_zebra'], default='iwildcam') - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='../iwildcam_v2.0/imgs/') - parser.add_argument('--split', type=str, default='val') - parser.add_argument('--seed', type=int, default=813765) - parser.add_argument('--ckpt-path', type=str, default=None, help='path to ckpt for restarting expt') - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - parser.add_argument('--use-subtree', action='store_true', help='use truncated OTT') - parser.add_argument('--batch_size', type=int, default=16) - - parser.add_argument('--kg-embed-model', choices=['distmult', 'conve'], default='distmult') - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - - # ConvE hyperparams - parser.add_argument('--embedding-shape1', type=int, default=20, help='The first dimension of the reshaped 2D embedding. The second dimension is infered. Default: 20') - parser.add_argument('--hidden-drop', type=float, default=0.3, help='Dropout for the hidden layer. Default: 0.3.') - parser.add_argument('--input-drop', type=float, default=0.2, help='Dropout for the input embeddings. Default: 0.2.') - parser.add_argument('--feat-drop', type=float, default=0.2, help='Dropout for the convolutional features. Default: 0.2.') - parser.add_argument('--use-bias', action='store_true', default=True, help='Use a bias in the convolutional layer. Default: True') - parser.add_argument('--hidden-size', type=int, default=9728, help='The side of the hidden layer. The required size changes with the size of the embeddings. Default: 9728 (embedding size 200).') - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - print(args.device) - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - if args.dataset == 'iwildcam': - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - entity_id_file = os.path.join(args.data_dir, 'entity2id_subtree.json') - else: - datacsv = pd.read_csv(os.path.join(args.data_dir, 'data_triples.csv'), low_memory=False) - entity_id_file = os.path.join(args.data_dir, 'entity2id.json') - - if not os.path.exists(entity_id_file): - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - if datacsv.iloc[i,1] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,0])))) - - if datacsv.iloc[i,-2] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,-3])))) - json.dump(entity2id, open(entity_id_file, 'w')) - else: - entity2id = json.load(open(entity_id_file, 'r')) - - num_ent_id = len(entity2id) - - print('len(entity2id) = {}'.format(len(entity2id))) - - target_list = generate_target_list(datacsv, entity2id) - - val_image_to_id_dataset = iWildCamOTTDataset(datacsv, args.split, args, entity2id, target_list, head_type="image", tail_type="id") - print('len(val_image_to_id_dataset) = {}'.format(len(val_image_to_id_dataset))) - - val_loader = DataLoader( - val_image_to_id_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=0, - pin_memory=True) - - model = MKGE(args, num_ent_id, target_list, args.device) - - model.to(args.device) - - # restore from ckpt - if args.ckpt_path: - ckpt = torch.load(args.ckpt_path, map_location=args.device) - model.load_state_dict(ckpt['model'], strict=False) - print('ckpt loaded...') - - evaluate(model, val_loader, target_list, args) diff --git a/gen_utils/analyze_img_loc.py b/gen_utils/analyze_img_loc.py deleted file mode 100644 index 194b4f5..0000000 --- a/gen_utils/analyze_img_loc.py +++ /dev/null @@ -1,219 +0,0 @@ -import matplotlib -import matplotlib.pyplot as plt -import pandas as pd -import argparse -import os -import numpy as np -import re -from tqdm import tqdm -import sklearn.cluster as cluster -from collections import defaultdict, Counter - -def getNumber(x): - return np.fromstring(x[1:-1], sep=' ', dtype=float) - -def plot_loc_viz(image_loc_dict, image_loc_dict_val, image_loc_dict_test): - plt.figure() - - # train - X_train, Y_train = [], [] - - for img_filename in tqdm(image_loc_dict): - x, y = getNumber(image_loc_dict[img_filename]) - X_train.append(x) - Y_train.append(y) - - scale = 200.0 * np.random.rand(len(X_train)) - plt.subplot(1, 4, 1) - plt.scatter(X_train, Y_train, label='train', s=scale, alpha=0.3) - plt.legend() - - # val - X_val, Y_val = [], [] - - for img_filename in tqdm(image_loc_dict_val): - x, y = getNumber(image_loc_dict_val[img_filename]) - X_val.append(x) - Y_val.append(y) - - scale = 200.0 * np.random.rand(len(X_val)) - plt.subplot(1, 4, 2) - plt.scatter(X_val, Y_val, label='val', s=scale, alpha=0.3) - plt.legend() - - # test - X_test, Y_test = [], [] - - for img_filename in tqdm(image_loc_dict_test): - x, y = getNumber(image_loc_dict_test[img_filename]) - X_test.append(x) - Y_test.append(y) - - scale = 200.0 * np.random.rand(len(X_test)) - plt.subplot(1, 4, 3) - plt.scatter(X_test, Y_test, label='test', s=scale, alpha=0.3) - plt.legend() - - plt.subplot(1, 4, 4) - scale = 200.0 * np.random.rand(len(X_train)) - plt.scatter(X_train, Y_train, label='train', s=scale, alpha=0.3) - scale = 200.0 * np.random.rand(len(X_val)) - plt.scatter(X_val, Y_val, label='val', s=scale, alpha=0.3) - scale = 200.0 * np.random.rand(len(X_test)) - plt.scatter(X_test, Y_test, label='test', s=scale, alpha=0.3) - plt.legend() - - plt.savefig('locs_splits.png') - -def plot_loc_hist(n_species_loc): - # plot histogram - plt.figure() - plt.hist(n_species_loc) - plt.xlabel('No. of species') - plt.ylabel('No. of locations') - plt.savefig('n_species_loc_hist.png') - -def create_image_id_dict(datacsv_id): - image_id_dict = {} - - for i in range(len(datacsv_id)): - image_filename = datacsv_id.iloc[i, 0] - specie_id = int(float(datacsv_id.iloc[i, -3])) - - image_id_dict[image_filename] = specie_id - - return image_id_dict - -def bhattacharyya_distance(distribution1, distribution2): - """ Estimate Bhattacharyya Distance (between General Distributions) - - Args: - distribution1: a sample distribution 1 - distribution2: a sample distribution 2 - - Returns: - Bhattacharyya distance - """ - sq = 0 - for i in range(len(distribution1)): - sq += np.sqrt(distribution1[i]*distribution2[i]) - - return -np.log(sq) - -def calc_dist_train_val(centroid_counters, centroid_counters_val, idx1, idx2): - counter_0_keys = list(set(centroid_counters[idx1].keys()) | set(centroid_counters_val[idx2].keys())) - - counter_0_train_dist, counter_0_val_dist = np.zeros(len(counter_0_keys)), np.zeros(len(counter_0_keys)) - - for item in centroid_counters[idx1]: - counter_0_train_dist[counter_0_keys.index(item)] += centroid_counters[idx1][item] - counter_0_train_dist = counter_0_train_dist/np.sum(counter_0_train_dist) - - for item in centroid_counters_val[idx2]: - counter_0_val_dist[counter_0_keys.index(item)] += centroid_counters_val[idx2][item] - counter_0_val_dist = counter_0_val_dist/np.sum(counter_0_val_dist) - - - counter_0_dist = bhattacharyya_distance(counter_0_train_dist, counter_0_val_dist) - return counter_0_dist - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='iwildcam_v2.0/') - parser.add_argument('--seed', type=int, default=813765) - args = parser.parse_args() - - np.random.seed(args.seed) - - mode = 'train' - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - - datacsv_loc = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'location') & (datacsv['split'] == mode), :] - datacsv_loc_val = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'location') & (datacsv['split'] == 'val'), :] - - datacsv_id = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'id') & (datacsv['split'] == mode), :] - datacsv_id_val = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'id') & (datacsv['split'] == 'val'), :] - - image_loc_dict, image_loc_dict_val, image_loc_dict_test = {}, {}, {} - loc_image_dict = defaultdict(list) - all_locs = set() - - for i in range(len(datacsv_loc)): - image_filename = datacsv_loc.iloc[i, 0] - loc = datacsv_loc.iloc[i, -3] - image_loc_dict[image_filename] = loc - loc_image_dict[loc].append(image_filename) - all_locs.add(loc) - - for i in range(len(datacsv_loc_val)): - image_filename = datacsv_loc_val.iloc[i, 0] - loc = datacsv_loc_val.iloc[i, -3] - image_loc_dict_val[image_filename] = loc - - all_locs = list(all_locs) - - assert len(image_loc_dict) == len(datacsv_loc) - - all_locs_arr = np.array(list(map(lambda x:getNumber(x), all_locs))) - - (centroid, label, _) = cluster.k_means(all_locs_arr, n_clusters=6) - - centroid_counters = [Counter() for _ in range(6)] - - image_id_dict = create_image_id_dict(datacsv_id) - image_id_dict_val = create_image_id_dict(datacsv_id_val) - - all_species = list(set(image_id_dict.values())) - - n_species = len(all_species) - - colors = np.random.rand(n_species) - - loc_species_dict = defaultdict(list) - - for img_filename in tqdm(image_loc_dict): - loc = image_loc_dict[img_filename] - species_id = image_id_dict[img_filename] - - loc_species_dict[loc].append(species_id) - - n_species_loc = [len(set(loc_species_dict[loc])) for loc in loc_species_dict] - - n_avg_species_loc = np.average(n_species_loc) - - for loc in tqdm(loc_species_dict): - centroid_counters[label[all_locs.index(loc)]].update(loc_species_dict[loc]) - - # plot locations of train/val/test - plot_loc_viz(image_loc_dict, image_loc_dict_val, image_loc_dict_test) - - centroid_counters_val = [Counter() for _ in range(6)] - - for img_filename in tqdm(image_loc_dict_val): - loc = getNumber(image_loc_dict_val[img_filename]) - - # find closest of (x, y) to each of train's centroid points - loc_dist = np.linalg.norm(loc - centroid, axis=-1) - cluster_id = np.argmin(loc_dist) - - # assign centroid label to this point - species_id = image_id_dict_val[img_filename] - centroid_counters_val[cluster_id].update([species_id]) - - - confus_mat = np.ones((6, 6))*np.inf - - for i in range(6): - for j in range(6): - if len(centroid_counters[i])>0 and len(centroid_counters_val[j])>0: - confus_mat[i, j] = calc_dist_train_val(centroid_counters, centroid_counters_val, i, j) - - - fig = plt.figure() - cax = plt.matshow(confus_mat) - fig.colorbar(cax) - plt.savefig('loc_corr_analysis.png') - - - - diff --git a/gen_utils/analyze_img_time.py b/gen_utils/analyze_img_time.py deleted file mode 100644 index 3173a38..0000000 --- a/gen_utils/analyze_img_time.py +++ /dev/null @@ -1,127 +0,0 @@ -import matplotlib -import matplotlib.pyplot as plt -import pandas as pd -import argparse -import os -import numpy as np -import re -from tqdm import tqdm -import sklearn.cluster as cluster -from collections import defaultdict, Counter -from re import match - -def getSeparated(item): - m = match(r"(.*)-(.*)-(.*) (.*):(.*):(\d{2})", item) - years, month, day, hour, minutes, second = m.groups() - return int(years), int(month), int(day), int(hour), int(minutes), int(second) - -def create_image_id_dict(datacsv_id): - image_id_dict = {} - - for i in range(len(datacsv_id)): - image_filename = datacsv_id.iloc[i, 0] - specie_id = int(float(datacsv_id.iloc[i, -3])) - - image_id_dict[image_filename] = specie_id - - return image_id_dict - -def bhattacharyya_distance(distribution1, distribution2): - """ Estimate Bhattacharyya Distance (between General Distributions) - - Args: - distribution1: a sample distribution 1 - distribution2: a sample distribution 2 - - Returns: - Bhattacharyya distance - """ - sq = 0 - for i in range(len(distribution1)): - sq += np.sqrt(distribution1[i]*distribution2[i]) - - return -np.log(sq) - -def calc_dist_train_val(centroid_counters, centroid_counters_val, idx1, idx2): - counter_0_keys = list(set(centroid_counters[idx1].keys()) | set(centroid_counters_val[idx2].keys())) - - counter_0_train_dist, counter_0_val_dist = np.zeros(len(counter_0_keys)), np.zeros(len(counter_0_keys)) - - for item in centroid_counters[idx1]: - counter_0_train_dist[counter_0_keys.index(item)] += centroid_counters[idx1][item] - counter_0_train_dist = counter_0_train_dist/np.sum(counter_0_train_dist) - - for item in centroid_counters_val[idx2]: - counter_0_val_dist[counter_0_keys.index(item)] += centroid_counters_val[idx2][item] - counter_0_val_dist = counter_0_val_dist/np.sum(counter_0_val_dist) - - counter_0_dist = bhattacharyya_distance(counter_0_train_dist, counter_0_val_dist) - return counter_0_dist - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--seed', type=int, default=813765) - args = parser.parse_args() - - np.random.seed(args.seed) - - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - - datacsv_time_train = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'time') & (datacsv['split'] == 'train'), :] - datacsv_time_val = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'time') & (datacsv['split'] == 'val'), :] - - - datacsv_id_train = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'id') & (datacsv['split'] == 'train'), :] - datacsv_id_val = datacsv.loc[(datacsv['datatype_h'] == 'image') & (datacsv['datatype_t'] == 'id') & (datacsv['split'] == 'val'), :] - - # compute distribution over hh, mm, ss - image_time_dict_train, image_time_dict_val = {}, {} - - for i in range(len(datacsv_time_train)): - image_filename = datacsv_time_train.iloc[i, 0] - time = datacsv_time_train.iloc[i, -3] - image_time_dict_train[image_filename] = time - - for i in range(len(datacsv_time_val)): - image_filename = datacsv_time_val.iloc[i, 0] - time = datacsv_time_val.iloc[i, -3] - image_time_dict_val[image_filename] = time - - image_id_dict_train = create_image_id_dict(datacsv_id_train) - image_id_dict_val = create_image_id_dict(datacsv_id_val) - - # calculate confus matrix for different hours - species_c_hour_train = [Counter() for _ in range(24)] - - for img_filename in tqdm(image_time_dict_train): - time = image_time_dict_train[img_filename] - yyyy, mon, dd, hh, mm, ss = getSeparated(time) - - species_id = image_id_dict_train[img_filename] - species_c_hour_train[hh].update([species_id]) - - species_c_hour_val = [Counter() for _ in range(24)] - - for img_filename in tqdm(image_time_dict_val): - time = image_time_dict_val[img_filename] - yyyy, mon, dd, hh, mm, ss = getSeparated(time) - - species_id = image_id_dict_val[img_filename] - species_c_hour_val[hh].update([species_id]) - - confus_mat = np.ones((24, 24))*np.inf - - for i in range(24): - for j in range(24): - if len(species_c_hour_train[i])>0 and len(species_c_hour_val[j])>0: - confus_mat[i, j] = calc_dist_train_val(species_c_hour_train, species_c_hour_val, i, j) - - - fig = plt.figure() - cax = plt.matshow(confus_mat) - fig.colorbar(cax) - plt.savefig('time_corr_analysis.png') - - - diff --git a/gen_utils/analyze_taxonomy_model.py b/gen_utils/analyze_taxonomy_model.py deleted file mode 100644 index 1f40c95..0000000 --- a/gen_utils/analyze_taxonomy_model.py +++ /dev/null @@ -1,362 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict, Counter -import math - -sys.path.append('../') - -from model import MKGE -from resnet import Resnet18, Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy, Recall, F1 -from PIL import Image -from dataset import iWildCamOTTDataset - -def level(a, node_parent_map): - if a not in node_parent_map: - return 0 - parent = node_parent_map[a] - return level(parent, node_parent_map)+1 - -def height(a, parent_node_map): - ans = -1 - - if a not in parent_node_map: - return 0 - - for child in parent_node_map[a]: - ans = max(ans, height(child, parent_node_map)) - - return ans+1 - - -def least_common_ancestor(a, b, node_parent_map): - - if level(a, node_parent_map) > level(b, node_parent_map): - a, b = b, a - - # if both are not at same level then move lower node upwards - d = level(b, node_parent_map) - level(a, node_parent_map) - - # node_parent_map[i] stores the parent of node i - while d > 0: - b = node_parent_map[b] - d-= 1 - - # base case if one was the ancestor of other node - if a == b: - return a - - # print('a = {}, b = {}'.format(a, b)) - if a not in node_parent_map or b not in node_parent_map: - return '805080' # return root as ancestor - - while node_parent_map[a] != node_parent_map[b]: - a = node_parent_map[a] - b = node_parent_map[b] - - # print('flag 1') - - return node_parent_map[a] - - -def print_ancestors(species_id, node_parent_map, target_list, taxon_id_to_name, overall_id_to_name): - out = [] - curr_node = species_id - - while True: - if str(curr_node) in taxon_id_to_name: - out.append(taxon_id_to_name[str(curr_node)]) - else: - out.append(overall_id_to_name[str(curr_node)]) - break - - if curr_node not in node_parent_map: - break - curr_node = node_parent_map[curr_node] - - print(' --> '.join(out)) - -def evaluate(model, val_loader, id2entity, overall_id_to_name, taxon_id_to_name, target_list, node_parent_map, parent_node_map, args): - model.eval() - torch.set_grad_enabled(False) - - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - correct_idx = [] - - avg_lca_height = 0 - total = 0 - - for labeled_batch in tqdm(val_loader): - h, r, t = labeled_batch - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'id')) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - y_true = detach_and_clone(batch_results['y_true']) - epoch_y_true.append(y_true) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - b_range = torch.arange(y_pred.size()[0], device=args.device) - - arg_outputs = torch.argsort(outputs, dim=-1, descending=True) - rank = 1 + torch.argsort(arg_outputs, dim=-1, descending=False)[b_range, y_true] - # print('rank = {}'.format(rank)) - - for i in range(y_true.size(0)): - if y_pred[i] == y_true[i]: - correct_idx.append(batch_idx * args.batch_size + i) - else: - lca = least_common_ancestor(int(id2entity[target_list[y_pred[i]].item()]), int(id2entity[target_list[y_true[i]].item()]), node_parent_map) - lca_height = height(lca, parent_node_map) - - avg_lca_height += lca_height - total += 1 - - epoch_y_pred.append(y_pred) - - batch_idx += 1 - if args.debug and batch_idx>10: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - Recall(prediction_fn=None, average='macro'), - F1(prediction_fn=None, average='macro'), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - print(f'Eval., split: {args.split}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100:.2f}, F1 macro: {results[metrics[2].agg_metric_field]*100:.2f}') - - avg_lca_height = avg_lca_height/total - - # print('total = {}'.format(total)) - # print('avg_lca_height = {}'.format(avg_lca_height)) - - return correct_idx, epoch_y_pred.tolist(), epoch_y_true.tolist(), avg_lca_height - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - # print('categories = {}'.format(categories)) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - -def check_list_equal(list_1, list_2): - for i in range(len(list_1)): - if list_1[i] != list_2[i]: - return False - return True - - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='../iwildcam_v2.0/imgs/') - parser.add_argument('--split', type=str, default='val') - parser.add_argument('--seed', type=int, default=813765) - - parser.add_argument('--ckpt-1-path', type=str, default=None, help='path to ckpt 1 for restarting expt') - parser.add_argument('--ckpt-2-path', type=str, default=None, help='path to ckpt 1 for restarting expt') - - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - parser.add_argument('--batch_size', type=int, default=16) - - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - parser.add_argument('--add-id-id', action='store_true', help='add idtoid triples in addition to other triples for training') - - parser.add_argument('--kg-embed-model', choices=['distmult', 'conve'], default='distmult') - - # ConvE hyperparams - parser.add_argument('--embedding-shape1', type=int, default=20, help='The first dimension of the reshaped 2D embedding. The second dimension is infered. Default: 20') - parser.add_argument('--hidden-drop', type=float, default=0.3, help='Dropout for the hidden layer. Default: 0.3.') - parser.add_argument('--input-drop', type=float, default=0.2, help='Dropout for the input embeddings. Default: 0.2.') - parser.add_argument('--feat-drop', type=float, default=0.2, help='Dropout for the convolutional features. Default: 0.2.') - parser.add_argument('--use-bias', action='store_true', default=True, help='Use a bias in the convolutional layer. Default: True') - parser.add_argument('--hidden-size', type=int, default=9728, help='The side of the hidden layer. The required size changes with the size of the embeddings. Default: 9728 (embedding size 200).') - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - - # construct OTT parent map - datacsv_id_id = datacsv.loc[(datacsv['datatype_h'] == 'id') & (datacsv['datatype_t'] == 'id')] - - node_parent_map = {} - parent_node_map = defaultdict(list) - - for idx in range(len(datacsv_id_id)): - node = int(float(datacsv.iloc[idx, 0])) - parent = int(float(datacsv.iloc[idx, -3])) - - node_parent_map[node] = parent - parent_node_map[parent].append(node) - - # print('node_parent_map = {}'.format(node_parent_map)) - # sys.exit(0) - - entity_id_file = os.path.join(args.data_dir, 'entity2id_subtree.json') - - if not os.path.exists(entity_id_file): - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - if datacsv.iloc[i,1] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,0])))) - - if datacsv.iloc[i,-2] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,-3])))) - json.dump(entity2id, open(entity_id_file, 'w')) - else: - entity2id = json.load(open(entity_id_file, 'r')) - - num_ent_id = len(entity2id) - - print('len(entity2id) = {}'.format(len(entity2id))) - - # print('entity2id = {}'.format(entity2id)) - id2entity = {v:k for k,v in entity2id.items()} - - target_list = generate_target_list(datacsv, entity2id) - # print('target_list = {}'.format(target_list)) - - val_image_to_id_dataset = iWildCamOTTDataset(datacsv, args.split, args, entity2id, target_list, head_type="image", tail_type="id") - print('len(val_image_to_id_dataset) = {}'.format(len(val_image_to_id_dataset))) - - val_loader = DataLoader( - val_image_to_id_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=0, - pin_memory=True) - - - model_1 = MKGE(args, num_ent_id, target_list, args.device) - model_2 = MKGE(args, num_ent_id, target_list, args.device) - - model_1.to(args.device) - model_2.to(args.device) - - overall_id_to_name = json.load(open(os.path.join(args.data_dir, 'overall_id_to_name.json'), 'r')) - taxon_id_to_name = json.load(open(os.path.join(args.data_dir, 'taxon_id_to_name.json'), 'r')) - - taxon_id_to_name['8032203'] = 'empty' - - # restore from ckpt - if args.ckpt_1_path: - ckpt = torch.load(args.ckpt_1_path, map_location=args.device) - model_1.load_state_dict(ckpt['model'], strict=False) - print('ckpt loaded...') - - if args.ckpt_2_path: - ckpt = torch.load(args.ckpt_2_path, map_location=args.device) - model_2.load_state_dict(ckpt['model'], strict=False) - print('ckpt loaded...') - - model_1_correct_idx, model_1_pred, model_1_true, lca_model_1 = evaluate(model_1, val_loader, id2entity, overall_id_to_name, taxon_id_to_name, target_list, node_parent_map, parent_node_map, args) - - model_2_correct_idx, model_2_pred, model_2_true, lca_model_2 = evaluate(model_2, val_loader, id2entity, overall_id_to_name, taxon_id_to_name, target_list, node_parent_map, parent_node_map, args) - - print('lca_model_1 = {}'.format(lca_model_1)) - print('lca_model_2 = {}'.format(lca_model_2)) - - # model_1 - model_2 - model_1_correct = list(set(model_1_correct_idx) - set(model_2_correct_idx)) - - # print('len(model_1_correct) = {}'.format(len(model_1_correct))) - - assert check_list_equal(model_1_true, model_2_true) - - # show taxonomy for cases where model_1 is correct but model_2 is not - - true_pred_c = Counter() - - for idx in model_1_correct: - model_1_true_label = model_1_true[idx] - model_1_pred_label = model_1_pred[idx] - model_2_pred_label = model_2_pred[idx] - - assert model_1_true_label == model_1_pred_label # model_1 is correct for this example, model_2 is incorrect - - print('true_label = {}, model_1_pred_label = {}, model_2_pred_label = {}'.format(overall_id_to_name[id2entity[target_list[model_1_true_label].item()]], overall_id_to_name[id2entity[target_list[model_1_pred_label].item()]], overall_id_to_name[id2entity[target_list[model_2_pred_label].item()]])) - - true_pred_c.update([(overall_id_to_name[id2entity[target_list[model_1_true_label].item()]], overall_id_to_name[id2entity[target_list[model_2_pred_label].item()]])]) - - # print taxonomy (list of ancestors) for y_true - print('ancestors of y_true: ') - print_ancestors(int(id2entity[target_list[model_1_true_label].item()]), node_parent_map, target_list, taxon_id_to_name, overall_id_to_name) - - print('ancestors of y_pred: ') - print_ancestors(int(id2entity[target_list[model_2_pred_label].item()]), node_parent_map, target_list, taxon_id_to_name, overall_id_to_name) - - print('\n') - - print(true_pred_c.most_common()) - - - - - diff --git a/gen_utils/dump_imageonly_pred_specie_wise.py b/gen_utils/dump_imageonly_pred_specie_wise.py deleted file mode 100644 index 8c1d744..0000000 --- a/gen_utils/dump_imageonly_pred_specie_wise.py +++ /dev/null @@ -1,236 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict -import math - -sys.path.append('../') - -from model import MKGE -from resnet import Resnet18, Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy -from PIL import Image -from dataset import iWildCamOTTDataset -import torchvision.transforms as transforms - -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN = [0.485, 0.456, 0.406] -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD = [0.229, 0.224, 0.225] - -def print_ancestors(species_id, node_parent_map, target_list, taxon_id_to_name, overall_id_to_name): - out = [] - curr_node = species_id - - while True: - if str(curr_node) in taxon_id_to_name: - out.append(taxon_id_to_name[str(curr_node)]) - else: - out.append(overall_id_to_name[str(curr_node)]) - break - - if curr_node not in node_parent_map: - break - curr_node = node_parent_map[curr_node] - - print(' --> '.join(out)) - -def evaluate(model, val_loader, args): - model.eval() - torch.set_grad_enabled(False) - - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - - y_pred_dict = {} - - for label_id in range(182): - y_pred_dict[label_id] = [] - - for labeled_batch in tqdm(val_loader): - x, y_true = labeled_batch - x = move_to(x, args.device) - y_true = move_to(y_true, args.device) - - outputs = model(x) - - batch_results = { - # 'g': g, - 'y_true': y_true.cpu(), - 'y_pred': outputs.cpu(), - # 'metadata': metadata, - } - - y_true = detach_and_clone(batch_results['y_true']) - epoch_y_true.append(y_true) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - - epoch_y_pred.append(y_pred) - - for i in range(y_true.size(0)): - x = (y_pred[i] == y_true[i]).long().item() - y_pred_dict[y_true[i].item()].append(x) # 1 means prediction matches label, 0 otherwise. Used for calculating F1 score. - - batch_idx += 1 - if args.debug: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - print(f'Eval., split: {args.split}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100:.2f}') - - return y_pred_dict - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - # print('categories = {}'.format(categories)) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - -class iWildCamDataset(Dataset): - def __init__(self, datacsv, root, img_dir, mode, entity2id, target_list): # dic_data <- datas - super(iWildCamDataset, self).__init__() - self.mode = mode - self.datacsv = datacsv.loc[datacsv['split'] == mode, :] - self.root = root - self.img_dir = img_dir - self.entity2id = entity2id - self.target_list = target_list - self.entity_to_species_id = {self.target_list[i, 0].item():i for i in range(len(self.target_list))} - - def __len__(self): - return len(self.datacsv) - - def __getitem__(self, idx): - y = torch.tensor([self.entity_to_species_id[self.entity2id[str(int(float(self.datacsv.iloc[idx, -3])))]]], dtype=torch.long).squeeze() - - img = Image.open(os.path.join(self.img_dir, self.datacsv.iloc[idx, 0])).convert('RGB') - - transform_steps = transforms.Compose([transforms.Resize((448, 448)), transforms.ToTensor(), transforms.Normalize(_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN, _DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD)]) - x = transform_steps(img) - - return x, y - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='../iwildcam_v2.0/imgs/') - parser.add_argument('--split', type=str, default='val') - parser.add_argument('--seed', type=int, default=813765) - parser.add_argument('--ckpt-path', type=str, default=None, help='path to ckpt for restarting expt') - parser.add_argument('--out-dir', type=str) - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - parser.add_argument('--use-subtree', action='store_true', help='use truncated OTT') - parser.add_argument('--batch_size', type=int, default=16) - - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - - # construct OTT parent map - datacsv_id_id = datacsv.loc[(datacsv['datatype_h'] == 'id') & (datacsv['datatype_t'] == 'id')] - node_parent_map = {} - - for idx in range(len(datacsv_id_id)): - node = int(float(datacsv.iloc[idx, 0])) - parent = int(float(datacsv.iloc[idx, -3])) - - node_parent_map[node] = parent - - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv')) - datacsv = datacsv.loc[(datacsv["datatype_h"] == "image") & (datacsv["datatype_t"] == "id")] - - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - _get_id(entity2id, str(int(float(datacsv.iloc[i,-3])))) - - print('len(entity2id) = {}'.format(len(entity2id))) - - target_list = generate_target_list(datacsv, entity2id) - - val_dataset = iWildCamDataset(datacsv, os.path.join('iwildcam_v2.0', 'imgs/'), args.img_dir, args.split, entity2id, target_list) - - id2entity = {v:k for k,v in entity2id.items()} - - val_loader = DataLoader( - val_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True) - - model = Resnet50(args) - model.to(args.device) - - # restore from ckpt - if args.ckpt_path: - ckpt = torch.load(args.ckpt_path) - model.load_state_dict(ckpt['model'], strict=False) - print('ckpt loaded...') - - y_pred_dict = evaluate(model, val_loader, args) - - json.dump(y_pred_dict, open(os.path.join(args.out_dir, 'y_pred_dict_{}.json'.format(args.split)), 'w')) - - diff --git a/gen_utils/dump_kge_pred_specie_wise.py b/gen_utils/dump_kge_pred_specie_wise.py deleted file mode 100644 index 121779b..0000000 --- a/gen_utils/dump_kge_pred_specie_wise.py +++ /dev/null @@ -1,211 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict -import math - -sys.path.append('../') - -from model import MKGE - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy, Recall, F1 -from PIL import Image -from dataset import iWildCamOTTDataset - -def evaluate(model, val_loader, target_list, node_parent_map, args): - model.eval() - torch.set_grad_enabled(False) - - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - - y_pred_dict = {} - - for label_id in range(182): - y_pred_dict[label_id] = [] - - for labeled_batch in tqdm(val_loader): - h, r, t = labeled_batch - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'id')) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - y_true = detach_and_clone(batch_results['y_true']) - epoch_y_true.append(y_true) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - for i in range(y_true.size(0)): - x = (y_pred[i] == y_true[i]).long().item() - y_pred_dict[y_true[i].item()].append(x) # 1 means prediction matches label, 0 otherwise. Used for calculating F1 score. - - batch_idx += 1 - if args.debug: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - Recall(prediction_fn=None, average='macro'), - F1(prediction_fn=None, average='macro'), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - print(f'Eval., split: {args.split}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100:.2f}, F1 macro: {results[metrics[2].agg_metric_field]*100:.2f}') - - return y_pred_dict - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - # print('categories = {}'.format(categories)) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='../iwildcam_v2.0/imgs/') - parser.add_argument('--split', type=str, default='val') - parser.add_argument('--seed', type=int, default=813765) - parser.add_argument('--ckpt-path', type=str, default=None, help='path to ckpt for restarting expt') - parser.add_argument('--out-dir', type=str) - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - parser.add_argument('--batch_size', type=int, default=16) - - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - - parser.add_argument('--kg-embed-model', choices=['distmult', 'conve'], default='distmult') - - # ConvE hyperparams - parser.add_argument('--embedding-shape1', type=int, default=20, help='The first dimension of the reshaped 2D embedding. The second dimension is infered. Default: 20') - parser.add_argument('--hidden-drop', type=float, default=0.3, help='Dropout for the hidden layer. Default: 0.3.') - parser.add_argument('--input-drop', type=float, default=0.2, help='Dropout for the input embeddings. Default: 0.2.') - parser.add_argument('--feat-drop', type=float, default=0.2, help='Dropout for the convolutional features. Default: 0.2.') - parser.add_argument('--use-bias', action='store_true', default=True, help='Use a bias in the convolutional layer. Default: True') - parser.add_argument('--hidden-size', type=int, default=9728, help='The side of the hidden layer. The required size changes with the size of the embeddings. Default: 9728 (embedding size 200).') - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - - # construct OTT parent map - datacsv_id_id = datacsv.loc[(datacsv['datatype_h'] == 'id') & (datacsv['datatype_t'] == 'id')] - node_parent_map = {} - - for idx in range(len(datacsv_id_id)): - node = int(float(datacsv.iloc[idx, 0])) - parent = int(float(datacsv.iloc[idx, -3])) - - node_parent_map[node] = parent - - entity_id_file = os.path.join(args.data_dir, 'entity2id_subtree.json') - - if not os.path.exists(entity_id_file): - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - if datacsv.iloc[i,1] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,0])))) - - if datacsv.iloc[i,-2] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,-3])))) - json.dump(entity2id, open(entity_id_file, 'w')) - else: - entity2id = json.load(open(entity_id_file, 'r')) - - num_ent_id = len(entity2id) - - print('len(entity2id) = {}'.format(len(entity2id))) - - # print('entity2id = {}'.format(entity2id)) - id2entity = {v:k for k,v in entity2id.items()} - - target_list = generate_target_list(datacsv, entity2id) - - val_image_to_id_dataset = iWildCamOTTDataset(datacsv, args.split, args, entity2id, target_list, head_type="image", tail_type="id") - print('len(val_image_to_id_dataset) = {}'.format(len(val_image_to_id_dataset))) - - val_loader = DataLoader( - val_image_to_id_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True) - - model = MKGE(args, num_ent_id, target_list, args.device) - - model.to(args.device) - - # restore from ckpt - if args.ckpt_path: - ckpt = torch.load(args.ckpt_path) - model.load_state_dict(ckpt['model'], strict=False) - print('ckpt loaded...') - - y_pred_dict = evaluate(model, val_loader, target_list, node_parent_map, args) - - json.dump(y_pred_dict, open(os.path.join(args.out_dir, 'y_pred_dict_{}.json'.format(args.split)), 'w')) - - diff --git a/gen_utils/eval_kge_specie_wise.py b/gen_utils/eval_kge_specie_wise.py deleted file mode 100644 index 6fea02d..0000000 --- a/gen_utils/eval_kge_specie_wise.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict -import math - -sys.path.append('../') - -from model import MKGE -from resnet import Resnet18, Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy -from PIL import Image -from dataset import iWildCamOTTDataset - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='../iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='../iwildcam_v2.0/imgs/') - parser.add_argument('--split', type=str, default='val') - parser.add_argument('--seed', type=int, default=813765) - - parser.add_argument('--y-pred-path-1', type=str, default=None, help='path to y_pred 1 predictions') - parser.add_argument('--y-pred-path-2', type=str, default=None, help='path to y_pred 2 predictions') - - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - parser.add_argument('--use-subtree', action='store_true', help='use truncated OTT') - parser.add_argument('--batch_size', type=int, default=16) - - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - y_1_pred_dict = json.load(open(args.y_pred_path_1)) - y_2_pred_dict = json.load(open(args.y_pred_path_2)) - - total = 0 - train_c = dict([(156, 48007), (1, 10267), (14, 7534), (0, 4078), (5, 4023), (2, 3986), (27, 3584), (54, 3177), (15, 3091), (30, 2740), (31, 2642), (57, 2401), (17, 1966), (12, 1913), (24, 1751), (158, 1709), (160, 1542), (48, 1530), (52, 1444), (32, 1428), (13, 1246), (155, 1168), (33, 1150), (11, 1042), (53, 977), (165, 949), (55, 904), (159, 865), (9, 771), (16, 730), (3, 716), (56, 684), (8, 605), (10, 538), (7, 531), (64, 459), (41, 457), (6, 450), (37, 433), (46, 380), (74, 367), (101, 350), (70, 290), (29, 243), (106, 201), (58, 200), (44, 194), (80, 190), (45, 180), (4, 161), (61, 158), (40, 146), (28, 136), (162, 128), (36, 117), (130, 110), (67, 108), (21, 106), (35, 102), (65, 100), (82, 100), (88, 92), (71, 87), (18, 81), (102, 80), (161, 80), (170, 80), (25, 75), (77, 73), (50, 70), (62, 62), (100, 60), (97, 60), (34, 55), (43, 50), (79, 48), (157, 46), (111, 44), (94, 39), (59, 38), (19, 38), (47, 36), (98, 32), (39, 30), (85, 30), (22, 29), (90, 29), (84, 29), (121, 28), (63, 25), (38, 24), (173, 23), (83, 21), (110, 21), (139, 20), (69, 20), (95, 19), (86, 18), (72, 18), (127, 15), (129, 15), (26, 15), (75, 15), (154, 15), (93, 14), (76, 13), (87, 13), (81, 13), (109, 12), (108, 12), (120, 12), (123, 12), (60, 12), (96, 12), (145, 11), (131, 10), (149, 10), (177, 10), (178, 10), (23, 9), (122, 9), (42, 9), (103, 9), (134, 9), (135, 9), (153, 9), (164, 9), (66, 8), (20, 8), (116, 8), (114, 7), (125, 7), (172, 7), (107, 6), (119, 6), (99, 6), (133, 6), (140, 6), (142, 6), (146, 6), (147, 6), (179, 6), (180, 6), (181, 6), (118, 5), (163, 5), (104, 4), (112, 4), (167, 4), (113, 3), (115, 3), (117, 3), (78, 3), (92, 3), (126, 3), (128, 3), (91, 3), (68, 3), (137, 3), (138, 3), (143, 3), (144, 3), (51, 3), (150, 3), (152, 3), (89, 2), (49, 2), (132, 2), (136, 2), (169, 2), (166, 2), (124, 1), (73, 1), (105, 1), (141, 1), (148, 1), (151, 1), (168, 1), (171, 1), (174, 1), (175, 1), (176, 1)]) - - threshold = 100 - - acc_1_avg = 0.0 - acc_2_avg = 0.0 - - for label_id in y_1_pred_dict: - # print(type(label_id)) - - if train_c[int(label_id)] <= threshold: - y_1_pred = y_1_pred_dict[label_id] - y_2_pred = y_2_pred_dict[label_id] - - assert len(y_1_pred)==len(y_2_pred) - - if len(y_1_pred)>0: - acc_1 = y_1_pred.count(1)*100.0/len(y_1_pred) - acc_2 = y_2_pred.count(1)*100.0/len(y_1_pred) - print(f'label_id = {label_id}, acc_1 = {acc_1:.2f}, acc_2 = {acc_2:.2f}, train_count = {train_c[int(label_id)]}') - - acc_1_avg += acc_1 - acc_2_avg += acc_2 - - total += 1 - - acc_1_avg = acc_1_avg/total - acc_2_avg = acc_2_avg/total - - print('acc_1_avg = {}'.format(acc_1_avg)) - print('acc_2_avg = {}'.format(acc_2_avg)) - - - diff --git a/main.py b/main.py deleted file mode 100644 index d779c2d..0000000 --- a/main.py +++ /dev/null @@ -1,570 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision -import sys -import json -from collections import defaultdict -import math -import torchvision.transforms as transforms -from torch.utils.tensorboard import SummaryWriter - - -from model import MKGE -from resnet import Resnet18, Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy -from PIL import Image -from dataset import iWildCamOTTDataset -from pytorchtools import EarlyStopping - -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN = [0.485, 0.456, 0.406] -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD = [0.229, 0.224, 0.225] - -################# -# image to id -################# - -def train_image_id(train_loader, model, optimizer, writer, args, epoch_id): - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - avg_loss_image_id = 0.0 - criterion_ce = nn.CrossEntropyLoss() - - for labeled_batch in tqdm(train_loader['image_to_id']): - h, r, t = labeled_batch - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'id')) - # outputs = model(h) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - # compute objective - loss = criterion_ce(batch_results['y_pred'], batch_results['y_true']) - batch_results['objective'] = loss.item() - loss.backward() - - avg_loss_image_id += loss.item() - - # update model and logs based on effective batch - optimizer.step() - model.zero_grad() - - epoch_y_true.append(detach_and_clone(batch_results['y_true'])) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - batch_idx += 1 - if args.debug: - break - - avg_loss_image_id = avg_loss_image_id/len(train_loader['image_to_id']) - print('train/avg_loss_image_id = {}'.format(avg_loss_image_id)) - writer.add_scalar('image_id_loss/train', avg_loss_image_id, epoch_id) - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - - results['epoch'] = epoch_id - print(f'Train epoch {epoch_id}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100.0:.2f}') - - writer.add_scalar('acc_image_id/train', results[metrics[0].agg_metric_field]*100.0, epoch_id) - -################# -# id to id -################# -def train_id_id(train_loader, model, optimizer, writer, args, epoch_id): - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - avg_loss_id_id = 0.0 - criterion_ce = nn.CrossEntropyLoss() - - for labeled_batch in tqdm(train_loader['id_to_id']): - h, r, t = labeled_batch - # print(h, r, t) - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('id', 'id')) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - # compute objective - loss = criterion_ce(batch_results['y_pred'], batch_results['y_true']) - avg_loss_id_id += loss.item() - - # print('loss = {}'.format(loss.item())) - batch_results['objective'] = loss.item() - loss.backward() - - # update model and logs based on effective batch - optimizer.step() - model.zero_grad() - - epoch_y_true.append(detach_and_clone(batch_results['y_true'])) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - batch_idx += 1 - if args.debug: - break - - avg_loss_id_id = avg_loss_id_id/len(train_loader['id_to_id']) - print('avg_loss_id_id = {}'.format(avg_loss_id_id)) - writer.add_scalar('avg_loss_id_id/train', avg_loss_id_id, epoch_id) - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - results['epoch'] = epoch_id - print(f'Train epoch {epoch_id}, id to id, Average acc: {results[metrics[0].agg_metric_field]*100.0:.2f}') - writer.add_scalar('acc_id_id/train', results[metrics[0].agg_metric_field]*100.0, epoch_id) - -################# -# image to location -################# -def train_image_location(train_loader, model, optimizer, writer, args, epoch_id): - batch_idx = 0 - avg_loss_image_location = 0.0 - criterion_bce = nn.BCEWithLogitsLoss() - - for labeled_batch in tqdm(train_loader['image_to_location']): - h, r, t = labeled_batch - - # print(h, r, t) - # print(t) - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'location')) - target = F.one_hot(t, num_classes=len(model.all_locs)).float() - loss = criterion_bce(outputs, target) - - avg_loss_image_location += loss.item() - - loss.backward() - - # update model and logs based on effective batch - optimizer.step() - model.zero_grad() - - batch_idx += 1 - if args.debug: - break - - avg_loss_image_location = avg_loss_image_location/len(train_loader['image_to_location']) - print('avg_loss_image_location = {}'.format(avg_loss_image_location)) - writer.add_scalar('avg_loss_image_location/train', avg_loss_image_location, epoch_id) - -################# -# image to time -################# -def train_image_time(train_loader, model, optimizer, writer, args, epoch_id): - batch_idx = 0 - avg_loss_image_time = 0.0 - criterion_ce = nn.CrossEntropyLoss() - criterion_bce = nn.BCEWithLogitsLoss() - - for labeled_batch in tqdm(train_loader['image_to_time']): - h, r, t = labeled_batch - - # print(h, r, t) - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'time')) - target = F.one_hot(t, num_classes=len(model.all_timestamps)).float() - loss = criterion_bce(outputs, target) - - avg_loss_image_time += loss.item() - - loss.backward() - - # update model and logs based on effective batch - optimizer.step() - model.zero_grad() - - batch_idx += 1 - if args.debug: - break - - avg_loss_image_time = avg_loss_image_time/len(train_loader['image_to_time']) - print('avg_loss_image_time = {}'.format(avg_loss_image_time)) - writer.add_scalar('avg_loss_image_time/train', avg_loss_image_time, epoch_id) - -def train(model, train_loader, optimizer, epoch_id, writer, args): - model.train() - torch.set_grad_enabled(True) - - if args.add_id_id: - train_id_id(train_loader, model, optimizer, writer, args, epoch_id) - - if args.add_image_location: - train_image_location(train_loader, model, optimizer, writer, args, epoch_id) - - if args.add_image_time and not args.add_id_id: - train_image_id(train_loader, model, optimizer, writer, args, epoch_id) - - if args.add_image_time: - train_image_time(train_loader, model, optimizer, writer, args, epoch_id) - - train_image_id(train_loader, model, optimizer, writer, args, epoch_id) - - return - -def evaluate(model, val_loader, optimizer, early_stopping, epoch_id, writer, args): - model.eval() - torch.set_grad_enabled(False) - criterion = nn.CrossEntropyLoss() - - epoch_y_true = [] - epoch_y_pred = [] - - batch_idx = 0 - avg_loss_image_id = 0.0 - for labeled_batch in tqdm(val_loader): - h, r, t = labeled_batch - h = move_to(h, args.device) - r = move_to(r, args.device) - t = move_to(t, args.device) - - outputs = model.forward_ce(h, r, t, triple_type=('image', 'id')) - - batch_results = { - 'y_true': t.cpu(), - 'y_pred': outputs.cpu(), - } - - batch_results['objective'] = criterion(batch_results['y_pred'], batch_results['y_true']).item() - avg_loss_image_id += batch_results['objective'] - - epoch_y_true.append(detach_and_clone(batch_results['y_true'])) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - batch_idx += 1 - if args.debug: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - results['epoch'] = epoch_id - - avg_loss_image_id = avg_loss_image_id/len(val_loader) - - early_stopping(-1*results[metrics[0].agg_metric_field], model, optimizer) - - print('val/avg_loss_image_id = {}'.format(avg_loss_image_id)) - writer.add_scalar('image_id_loss/val', avg_loss_image_id, epoch_id) - - writer.add_scalar('acc_image_id/val', results[metrics[0].agg_metric_field]*100, epoch_id) - - print(f'Eval. epoch {epoch_id}, image to id, Average acc: {results[metrics[0].agg_metric_field]*100:.2f}') - - return results, epoch_y_pred - - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - # print('categories = {}'.format(categories)) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument('--dataset', choices=['iwildcam', 'mountain_zebra'], default='iwildcam') - parser.add_argument('--data-dir', type=str, default='iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='iwildcam_v2.0/imgs/') - parser.add_argument('--batch_size', type=int, default=16) - parser.add_argument('--n_epochs', type=int, default=12) - parser.add_argument('--img-lr', type=float, default=3e-5, help='lr for img embed params') - parser.add_argument('--lr', type=float, default=1e-3, help='default lr for all parameters') - parser.add_argument('--loc-lr', type=float, default=1e-3, help='lr for location embedding') - parser.add_argument('--time-lr', type=float, default=1e-3, help='lr for time embedding') - parser.add_argument('--weight_decay', type=float, default=0.0) - parser.add_argument('--device', type=int, nargs='+', default=[0]) - parser.add_argument('--seed', type=int, default=813765) - parser.add_argument('--save-dir', type=str, default='ckpts/toy/') - parser.add_argument('--ckpt-path', type=str, default=None, help='path to ckpt for restarting expt') - parser.add_argument('--start-epoch', type=int, default=0, help='epoch id to restore model') - parser.add_argument('--early-stopping-patience', type=int, default=5, help='early stop if metric does not improve for x epochs') - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no-cuda', action='store_true') - - parser.add_argument('--kg-embed-model', choices=['distmult', 'conve'], default='distmult') - parser.add_argument('--embedding-dim', type=int, default=512) - parser.add_argument('--location_input_dim', type=int, default=2) - parser.add_argument('--time_input_dim', type=int, default=1) - parser.add_argument('--mlp_location_numlayer', type=int, default=3) - parser.add_argument('--mlp_time_numlayer', type=int, default=3) - - parser.add_argument('--img-embed-model', choices=['resnet18', 'resnet50'], default='resnet50') - parser.add_argument('--use-data-subset', action='store_true') - parser.add_argument('--subset-size', type=int, default=10) - - parser.add_argument('--add-id-id', action='store_true', help='add idtoid triples in addition to other triples for training') - parser.add_argument('--add-image-location', action='store_true', help='add imagetolocation triples in addition to other triples for training') - parser.add_argument('--add-image-time', action='store_true', help='use only imagetotime triples in addition to other triples for training') - parser.add_argument('--omit-double-img-id', action='store_true', help='omit double image id after location') - - # ConvE hyperparams - parser.add_argument('--embedding-shape1', type=int, default=20, help='The first dimension of the reshaped 2D embedding. The second dimension is infered. Default: 20') - parser.add_argument('--hidden-drop', type=float, default=0.3, help='Dropout for the hidden layer. Default: 0.3.') - parser.add_argument('--input-drop', type=float, default=0.2, help='Dropout for the input embeddings. Default: 0.2.') - parser.add_argument('--feat-drop', type=float, default=0.2, help='Dropout for the convolutional features. Default: 0.2.') - parser.add_argument('--use-bias', action='store_true', default=True, help='Use a bias in the convolutional layer. Default: True') - parser.add_argument('--hidden-size', type=int, default=9728, help='The side of the hidden layer. The required size changes with the size of the embeddings. Default: 9728 (embedding size 200).') - - args = parser.parse_args() - - print('args = {}'.format(args)) - args.device = torch.device('cuda') if not args.no_cuda and torch.cuda.is_available() else torch.device('cpu') - - # Set random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - writer = SummaryWriter(log_dir=args.save_dir) - - if args.dataset == 'iwildcam': - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - entity_id_file = os.path.join(args.data_dir, 'entity2id_subtree.json') - else: - datacsv = pd.read_csv(os.path.join(args.data_dir, 'data_triples.csv'), low_memory=False) - entity_id_file = os.path.join(args.data_dir, 'entity2id.json') - - - if not os.path.exists(entity_id_file): - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - if datacsv.iloc[i,1] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,0])))) - - if datacsv.iloc[i,4] == "id": - _get_id(entity2id, str(int(float(datacsv.iloc[i,3])))) - json.dump(entity2id, open(entity_id_file, 'w')) - else: - entity2id = json.load(open(entity_id_file, 'r')) - - num_ent_id = len(entity2id) - - print('len(entity2id) = {}'.format(len(entity2id))) - - target_list = generate_target_list(datacsv, entity2id) - - train_image_to_id_dataset = iWildCamOTTDataset(datacsv, 'train', args, entity2id, target_list, head_type="image", tail_type="id") - print('len(train_image_to_id_dataset) = {}'.format(len(train_image_to_id_dataset))) - - if args.add_id_id: - train_id_to_id_dataset = iWildCamOTTDataset(datacsv, 'train', args, entity2id, target_list, head_type="id", tail_type="id") - print('len(train_id_to_id_dataset) = {}'.format(len(train_id_to_id_dataset))) - - if args.add_image_location: - train_image_to_location_dataset = iWildCamOTTDataset(datacsv, 'train', args, entity2id, target_list, head_type="image", tail_type="location") - print('len(train_image_to_location_dataset) = {}'.format(len(train_image_to_location_dataset))) - - if args.add_image_time: - train_image_to_time_dataset = iWildCamOTTDataset(datacsv, 'train', args, entity2id, target_list, head_type="image", tail_type="time") - print('len(train_image_to_time_dataset) = {}'.format(len(train_image_to_time_dataset))) - - val_image_to_id_dataset = iWildCamOTTDataset(datacsv, 'val', args, entity2id, target_list, head_type="image", tail_type="id") - print('len(val_image_to_id_dataset) = {}'.format(len(val_image_to_id_dataset))) - - model_kwargs = {} - if args.kg_embed_model == 'conve': - model_kwargs['drop_last'] = True - - train_loader_image_to_id = DataLoader( - train_image_to_id_dataset, - shuffle=True, # Shuffle training dataset - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True, - **model_kwargs) - - if args.add_id_id: - train_loader_id_to_id = DataLoader( - train_id_to_id_dataset, - shuffle=True, # Shuffle training dataset - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True, - **model_kwargs) - - if args.add_image_location: - train_loader_image_to_location = DataLoader( - train_image_to_location_dataset, - shuffle=True, # Shuffle training dataset - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True, - **model_kwargs) - - if args.add_image_time: - train_loader_image_to_time = DataLoader( - train_image_to_time_dataset, - shuffle=True, # Shuffle training dataset - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True, - **model_kwargs) - - train_loaders = {} - - train_loaders['image_to_id'] = train_loader_image_to_id - - if args.add_id_id: - train_loaders['id_to_id'] = train_loader_id_to_id - - if args.add_image_location: - train_loaders['image_to_location'] = train_loader_image_to_location - - if args.add_image_time: - train_loaders['image_to_time'] = train_loader_image_to_time - - val_loader = DataLoader( - val_image_to_id_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True) - - kwargs = {} - if args.add_image_time: - kwargs['all_timestamps'] = train_image_to_time_dataset.all_timestamps - if args.add_image_location: - kwargs['all_locs'] = train_image_to_location_dataset.all_locs - - model = MKGE(args, num_ent_id, target_list, args.device, **kwargs) - - model.to(args.device) - - early_stopping = EarlyStopping(patience=args.early_stopping_patience, verbose=True, ckpt_path=os.path.join(args.save_dir, 'model.pt'), best_ckpt_path=os.path.join(args.save_dir, 'best_model.pt')) - - params_diff_lr = ['ent_embedding', 'rel_embedding', 'image_embedding', 'location_embedding', 'time_embedding'] - - optimizer_grouped_parameters = [ - {"params": [param for p_name, param in model.named_parameters() if not any([x in p_name for x in params_diff_lr])]}, - {"params": model.ent_embedding.parameters(), "lr": args.lr}, - {"params": model.rel_embedding.parameters(), "lr": args.lr}, - {"params": model.image_embedding.parameters(), "lr": args.img_lr}, - {"params": model.location_embedding.parameters(), "lr": args.loc_lr}, - {"params": model.time_embedding.parameters(), "lr": args.time_lr}, - ] - - optimizer = optim.Adam( - optimizer_grouped_parameters, - lr=args.lr, - weight_decay=args.weight_decay) - - # restore from ckpt - if args.ckpt_path: - print('ckpt loaded...') - ckpt = torch.load(args.ckpt_path) - model.load_state_dict(ckpt['model'], strict=False) - optimizer.load_state_dict(ckpt['dense_optimizer']) - - for epoch_id in range(args.start_epoch, args.n_epochs): - print('\nEpoch [%d]:\n' % epoch_id) - - # First run training - train(model, train_loaders, optimizer, epoch_id, writer, args) - - # Then run val - val_results, y_pred = evaluate(model, val_loader, optimizer, early_stopping, epoch_id, writer, args) - - if early_stopping.early_stop: - print("Early stopping...") - break - - writer.close() - -if __name__=='__main__': - main() diff --git a/model.py b/model.py deleted file mode 100644 index 7719cc1..0000000 --- a/model.py +++ /dev/null @@ -1,191 +0,0 @@ -import torch -import torch.nn as nn -from torch.nn import functional as F -from torch import Tensor -from typing import Tuple - -from torchvision.models import resnet18, resnet50 -from torchvision.models import ResNet18_Weights, ResNet50_Weights - -class MKGE(nn.Module): - def __init__(self, args, num_ent_uid, target_list, device, all_locs=None, num_habitat=None, all_timestamps=None): - super(MKGE, self).__init__() - self.args = args - self.num_ent_uid = num_ent_uid - - self.num_relations = 4 - - self.ent_embedding = torch.nn.Embedding(self.num_ent_uid, args.embedding_dim, sparse=False) - self.rel_embedding = torch.nn.Embedding(self.num_relations, args.embedding_dim, sparse=False) - - if self.args.kg_embed_model == 'conve': - self.inp_drop = torch.nn.Dropout(args.input_drop) - self.hidden_drop = torch.nn.Dropout(args.hidden_drop) - self.feature_map_drop = torch.nn.Dropout2d(args.feat_drop) - - self.emb_dim1 = args.embedding_shape1 # important parameter for ConvE - self.emb_dim2 = args.embedding_dim // self.emb_dim1 - - self.conv1 = torch.nn.Conv2d(1, 32, (3, 3), 1, 0, bias=args.use_bias) - self.bn0 = torch.nn.BatchNorm2d(1) - self.bn1 = torch.nn.BatchNorm2d(32) - self.bn2 = torch.nn.BatchNorm1d(args.embedding_dim) - self.fc = torch.nn.Linear(args.hidden_size, args.embedding_dim) - - self.location_embedding = MLP(args.location_input_dim, args.embedding_dim, args.mlp_location_numlayer) - - self.time_embedding = MLP(args.time_input_dim, args.embedding_dim, args.mlp_time_numlayer) - - if self.args.img_embed_model == 'resnet50': - self.image_embedding = resnet50(weights=ResNet50_Weights.IMAGENET1K_V1) - self.image_embedding.fc = nn.Linear(2048, args.embedding_dim) - else: - self.image_embedding = resnet18(weights=ResNet18_Weights.IMAGENET1K_V1) - self.image_embedding.fc = nn.Linear(512, args.embedding_dim) - - self.target_list = target_list - if all_locs is not None: - self.all_locs = all_locs.to(device) - if all_timestamps is not None: - self.all_timestamps = all_timestamps.to(device) - - self.args = args - self.device = device - - self.init() - - def init(self): - nn.init.xavier_uniform_(self.ent_embedding.weight.data) - nn.init.xavier_uniform_(self.rel_embedding.weight.data) - nn.init.xavier_uniform_(self.image_embedding.fc.weight.data) - - def forward(self, h, r, t): - - emb_h = self.batch_embedding_concat_h(h) - - emb_r = self.rel_embedding(r.squeeze(-1)) - - if self.args.kg_embed_model == 'distmult': - emb_t = self.batch_embedding_concat_h(t) - score = torch.sum(emb_h * emb_r * emb_t, -1) - - elif self.args.kg_embed_model == 'conve': - e1_embedded = e1_embedded.view(-1, 1, self.emb_dim1, self.emb_dim2) # [batch, 1, emb_dim1, emb_dim2] - rel_embedded = rel_embedded.view(-1, 1, self.emb_dim1, self.emb_dim2) # [batch, 1, emb_dim1, emb_dim2] - - stacked_inputs = torch.cat([e1_embedded, rel_embedded], 2) # [batch, 1, 2*emb_dim1, emb_dim2] - - stacked_inputs = self.bn0(stacked_inputs) - x = self.inp_drop(stacked_inputs) - x = self.conv1(x) - x = self.bn1(x) - x = F.relu(x) - x = self.feature_map_drop(x) - x = x.view(x.shape[0], -1) - x = self.fc(x) - x = self.hidden_drop(x) - x = self.bn2(x) - x = F.relu(x) - score = x * t - else: - raise NotImplementedError - - return score - - # @profile - def forward_ce(self, h, r, t, triple_type=None): - emb_h = self.batch_embedding_concat_h(h) # [batch, hid] - - emb_r = self.rel_embedding(r.squeeze(-1)) # [batch, hid] - - if self.args.kg_embed_model == 'distmult': - emb_hr = emb_h * emb_r # [batch, hid] - elif self.args.kg_embed_model == 'conve': - emb_h = emb_h.view(-1, 1, self.emb_dim1, self.emb_dim2) # [batch, 1, emb_dim1, emb_dim2] - emb_r = emb_r.view(-1, 1, self.emb_dim1, self.emb_dim2) # [batch, 1, emb_dim1, emb_dim2] - - stacked_inputs = torch.cat([emb_h, emb_r], 2) # [batch, 1, 2*emb_dim1, emb_dim2] - - stacked_inputs = self.bn0(stacked_inputs) - x = self.inp_drop(stacked_inputs) - x = self.conv1(x) - x = self.bn1(x) - x = F.relu(x) - x = self.feature_map_drop(x) - x = x.view(x.shape[0], -1) - x = self.fc(x) - x = self.hidden_drop(x) - x = self.bn2(x) - emb_hr = F.relu(x) - else: - raise NotImplementedError - - if triple_type == ('image', 'id'): - score = torch.mm(emb_hr, self.ent_embedding.weight[self.target_list.squeeze(-1)].T) # [batch, n_ent] - elif triple_type == ('id', 'id'): - score = torch.mm(emb_hr, self.ent_embedding.weight.T) # [batch, n_ent] - elif triple_type == ('image', 'location'): - loc_emb = self.location_embedding(self.all_locs) # computed for each batch - score = torch.mm(emb_hr, loc_emb.T) - elif triple_type == ('image', 'time'): - time_emb = self.time_embedding(self.all_timestamps) - score = torch.mm(emb_hr, time_emb.T) - else: - raise NotImplementedError - - return score - - def batch_embedding_concat_h(self, e1): - e1_embedded = None - - if len(e1.size())==1 or e1.size(1) == 1: # uid - # print('ent_embedding = {}'.format(self.ent_embedding.weight.size())) - e1_embedded = self.ent_embedding(e1.squeeze(-1)) - elif e1.size(1) == 15: # time - e1_embedded = self.time_embedding(e1) - elif e1.size(1) == 2: # GPS - e1_embedded = self.location_embedding(e1) - elif e1.size(1) == 3: # Image - e1_embedded = self.image_embedding(e1) - - return e1_embedded - - -class MLP(nn.Module): - def __init__(self, - input_dim, - output_dim, - num_layers=3, - p_dropout=0.0, - bias=True): - - super().__init__() - - self.input_dim = input_dim - self.output_dim = output_dim - - self.p_dropout = p_dropout - step_size = (input_dim - output_dim) // num_layers - hidden_dims = [output_dim + (i * step_size) - for i in reversed(range(num_layers))] - - mlp = list() - layer_indim = input_dim - for hidden_dim in hidden_dims: - mlp.extend([nn.Linear(layer_indim, hidden_dim, bias), - nn.Dropout(p=self.p_dropout, inplace=True), - nn.PReLU()]) - - layer_indim = hidden_dim - - self.mlp = nn.Sequential(*mlp) - - # initialize weights - self.init() - - def forward(self, x): - return self.mlp(x) - - def init(self): - for param in self.parameters(): - nn.init.uniform_(param) diff --git a/preprocess_data_iwildcam.py b/preprocess_data_iwildcam.py deleted file mode 100644 index b6ee0aa..0000000 --- a/preprocess_data_iwildcam.py +++ /dev/null @@ -1,207 +0,0 @@ -from wilds import get_dataset -import pandas as pd -import numpy as np -import json -from tqdm import tqdm - -def gps(x): - return np.array([x["latitude"], x["longitude"]]) - -# Load the full dataset, and download it if necessary -dataset = get_dataset(dataset="iwildcam", download=True) - -metadata = pd.read_csv("data/iwildcam_v2.0/metadata.csv") -categories = pd.read_csv("data/iwildcam_v2.0/categories.csv") - -# the map is iwildcam_id_to_name {y:name,....} -k = list(categories.y) -v = list(categories.name) - -iwildcam_id_to_name = {} -for i in range(len(k)): - iwildcam_id_to_name[k[i]] = v[i] - -iwildcam_name_to_id = {v:k for k,v in iwildcam_id_to_name.items()} - -# the map processing (replaces iwildcam category ids by species names) -metadata_y = list(metadata.y) -for i in range(len(metadata_y)): - metadata_y[i] = iwildcam_id_to_name[metadata_y[i]] - -metadata.y = metadata_y -metadata = metadata.loc[:, ["split", "location", "y", "datetime", "filename"]] -metadata.columns = ["split", "location", "name", "datetime", "filename"] - -# store time used data -time_used = metadata -# load pre_used_taxonomy.csv to get dic{name:uid} -taxon = pd.read_csv("ott_taxonomy.csv") - -k = list(taxon.name) -v = list(taxon.uid) - -# the map is taxon_name_to_id {name:uid,....} -taxon_name_to_id = {} -for i in range(len(k)): - taxon_name_to_id[k[i]] = v[i] - -taxon_id_to_name = {x:y for x,y in zip(taxon.uid, taxon.name)} -json.dump(taxon_id_to_name, open('data/iwildcam_v2.0/taxon_id_to_name.json', 'w'), indent=1) - -category_offset_non_intersection = max(taxon_name_to_id.values()) + 1 - - -meta_categories = list(set([x for x in metadata.name])) - -ott_categories = list(taxon.name) - -intersection_categories = list(set(ott_categories) & set(meta_categories)) - -# intersection of iwildcam and OTT -metadata_intersection = metadata.loc[metadata["name"].isin(intersection_categories), :].copy() - -# non-interesection part -metadata_non_intersection = metadata.loc[~metadata["name"].isin(intersection_categories), :].copy() - -# replace name by uid in metadata_intersection -metadata_name = list(metadata_intersection.name) -for i in range(len(metadata_name)): - metadata_name[i] = taxon_name_to_id[metadata_name[i]] - -metadata_intersection.name = metadata_name -metadata_intersection.columns = ["split", "location", "uid", "datetime", "filename"] - -metadata_non_intersection_name = list(metadata_non_intersection.name) -non_intersection_uids = set() -overall_id_to_name = {} - -for i in range(len(metadata_non_intersection_name)): - specie_name = metadata_non_intersection_name[i] - metadata_non_intersection_name[i] = iwildcam_name_to_id[specie_name] + category_offset_non_intersection - non_intersection_uids.add(iwildcam_name_to_id[specie_name]) - overall_id_to_name[metadata_non_intersection_name[i]] = specie_name - -metadata_non_intersection.name = metadata_non_intersection_name - -intersection_uids = set([iwildcam_name_to_id[taxon_id_to_name[x]] for x in metadata_intersection.uid]) - -for specie_id in intersection_uids: - overall_id_to_name[taxon_name_to_id[iwildcam_id_to_name[specie_id]]] = iwildcam_id_to_name[specie_id] - -common = non_intersection_uids & intersection_uids -common = [iwildcam_id_to_name[x] for x in common] - -json.dump(overall_id_to_name, open('data/iwildcam_v2.0/overall_id_to_name.json', 'w')) - -# re-name name column -metadata_non_intersection.columns = ["split", "location", "uid", "datetime", "filename"] - -# concatenate metadata_intersection and metadata_non_intersection -metadata = pd.concat([metadata_intersection, metadata_non_intersection]) - -# store uid used dataset -uid_used = metadata - -gps_data = pd.read_json('gps_locations.json') -gps_data = gps_data.transpose() -gps_data.insert(loc=2, column="location", value=gps_data.index.to_list()) -gps_data = gps_data.sort_index(ascending=True) - -gps_data["GPS"] = gps_data.apply(gps, axis=1) - -k = list(gps_data.location) -v = list(gps_data.GPS) - -# find the species that have GPS in metadata -metadata = metadata.loc[metadata["location"].isin(k), :].copy() - - -# the map is dic {location:GPS,....} -dic = {} -for i in range(len(k)): - dic[k[i]] = v[i] - -# make location to GPS -metadata_location = list(metadata.location) -for i in range(len(metadata_location)): - metadata_location[i] = dic[metadata_location[i]] - -metadata.location = metadata_location - -# store GPS used data -gps_used = metadata - -taxon = taxon.fillna(0) -taxon = taxon.loc[:, ["uid", "parent_uid"]] -taxon.columns = ["h", "t"] -taxon.insert(loc=1, column="r", value=1) -taxon.insert(loc=1, column="datatype_h", value="id") -taxon.insert(loc=4, column="datatype_t", value="id") -taxon.insert(loc=5, column="split", value="train") -taxon.columns = ["h", "datatype_h", "r", "t", "datatype_t", "split"] - -takeLocation = gps_used.loc[:, ["filename", "location", "split"]] -takeLocation.insert(loc=1, column="r", value=2) -takeLocation.insert(loc=1, column="datatype_h", value="image") -takeLocation.insert(loc=4, column="datatype_t", value="location") -takeLocation.columns = ["h", "datatype_h", "r", "t", "datatype_t", "split"] - -takeTime = time_used.loc[:, ["filename", "datetime", "split"]] -takeTime.insert(loc=1, column="r", value=0) -takeTime.insert(loc=1, column="datatype_h", value="image") -takeTime.insert(loc=4, column="datatype_t", value="time") -takeTime.columns = ["h", "datatype_h", "r", "t", "datatype_t", "split"] - -imageIsIn = uid_used.loc[:, ["filename", "uid", "split"]] -imageIsIn.insert(loc=1, column="r", value=3) -imageIsIn.insert(loc=1, column="datatype_h", value="image") -imageIsIn.insert(loc=4, column="datatype_t", value="id") -imageIsIn.columns = ["h", "datatype_h", "r", "t", "datatype_t", "split"] - -a = pd.concat([taxon, imageIsIn], ignore_index=True) -a = pd.concat([a, takeTime], ignore_index=True) -a = pd.concat([a, takeLocation], ignore_index=True) - -inner = a.loc[(a["datatype_h"]=="image") & (a["datatype_t"]=="id"),:].copy() - -ott = a.loc[(a["datatype_h"]=="id") & (a["datatype_t"]=="id"),:].copy() - -son = list(ott["h"]) -father = list(ott["t"]) -paths = {} -for i in tqdm(range(len(son))): - paths[int(float(son[i]))] = int(float(father[i])) - -leaf_node = list(inner.t) -leaf_nodes = [] -for item in tqdm(leaf_node): - if int(float(item)) not in leaf_nodes: - leaf_nodes.append(int(float(item))) - -list_paths = [] -def get_paths(leaf_node, paths, nodes_list): - while leaf_node in paths.keys(): - # print(leaf_node,"->",paths[leaf_node]) - nodes_list.append(leaf_node) - leaf_node = paths[leaf_node] - -def get_path_nodes(leaf_nodes, paths): - nodes_list = [] - for item in leaf_nodes: - get_paths(item,paths,nodes_list) - return nodes_list - -paths_nodes = get_path_nodes(leaf_nodes, paths) - -ott["h"] = paths.keys() -ott["t"] = paths.values() - -ott = ott.loc[(ott['h'].isin(paths_nodes)) & (ott['t'].isin(paths_nodes)),:] -ott = ott.reset_index() - -a = a.loc[(a["datatype_h"] != "id"),:] -a.reset_index() - -dataset = pd.concat([ott, a], ignore_index=True) -dataset = dataset.iloc[:,1:] -dataset.to_csv("data/iwildcam_v2.0/dataset_subtree.csv",index = False) diff --git a/preprocess_data_mountain_zebra.py b/preprocess_data_mountain_zebra.py deleted file mode 100644 index d9f6289..0000000 --- a/preprocess_data_mountain_zebra.py +++ /dev/null @@ -1,227 +0,0 @@ -import pandas as pd -import numpy as np -import json -from tqdm import tqdm -import argparse -import random, string -import os - - -if __name__=='__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--data-dir', type=str, default='data/snapshot_mountain_zebra/') - parser.add_argument('--use-loc-canonical-id', action='store_true') - parser.add_argument('--no-drop-nonexist-imgs', action='store_true') - parser.add_argument('--split-dataset', action='store_true', help='randomly split into train/val/test splits') - parser.add_argument('--no-datetime', action='store_true', help='ignore date/time') - parser.add_argument('--no-location', action='store_true', help='ignore location') - parser.add_argument('--species-common-names-file', type=str, default='data/snapshot_mountain_zebra/category_to_label_map.json') - parser.add_argument('--img-prefix', type=str, default='') - parser.add_argument('--dataset-prefix', type=str, default='') - - - args = parser.parse_args() - - annot_file = os.path.join(args.data_dir, 'annotations.json') - loc_file = os.path.join(args.data_dir, 'locations.csv') - category_to_label_map = json.load(open(args.species_common_names_file, 'r')) - - annotations_json = json.load(open(annot_file)) - - taxon_id_to_name_filename = 'snapshot_mountain_zebra/taxon_id_to_name_lila.json' - - taxon_id_to_name = json.load(open(taxon_id_to_name_filename, 'r')) - taxon_name_to_id = {v:k for k,v in taxon_id_to_name.items()} - - print('len(taxon_name_to_id) = {}'.format(len(taxon_name_to_id))) - print('len(taxon_id_to_name) = {}'.format(len(taxon_id_to_name))) - - if os.path.exists(loc_file) and not args.use_loc_canonical_id: - location_coordinates = pd.read_csv(loc_file) - else: - location_coordinates = None - - img_json = annotations_json['images'] - img_json = [x for x in img_json if args.no_drop_nonexist_imgs or os.path.exists(os.path.join(args.data_dir, args.img_prefix, x['file_name']))] - # print(img_json[0].keys()) - - # add y labels to metadata - annotations = annotations_json['annotations'] - - annotations_image_id = [x['image_id'] for x in annotations] - annotations_category_id = [x['category_id'] for x in annotations] - - annotations_df = pd.DataFrame(list(zip(annotations_image_id, annotations_category_id)), columns=['image_id', 'category_id']) - - metadata = annotations_df - - if 'caltech' in args.data_dir: - datetime_field = 'date_captured' - else: - datetime_field = 'datetime' - - # add image filename - img_ids = [x['id'] for x in tqdm(img_json)] - img_filenames = [(args.img_prefix + x['file_name']) for x in img_json] - - img_loc = [x['location'] for x in img_json] - - if not args.no_datetime: - img_datetime = [x[datetime_field] for x in img_json] - img_df = pd.DataFrame(list(zip(img_ids, img_filenames, img_loc, img_datetime)), columns=['image_id', 'filename', 'location', 'datetime']) - else: - img_df = pd.DataFrame(list(zip(img_ids, img_filenames, img_loc)), columns=['image_id', 'filename', 'location']) - - # construct a df with location paired to split - - # TODO; check if list of locations is in order - locs = list(img_df.location) - splits = [] - - split_json_file = open(os.path.join(args.data_dir, 'splits.json')) - split_json = json.load(split_json_file) - - train_locs = set(split_json['splits']['train']) - val_locs = set(split_json['splits']['val']) - - if 'test' in split_json['splits']: - test_locs = set(split_json['splits']['test']) - else: - test_locs = set() - - for loc in locs: - if loc in train_locs: - splits.append('train') - elif loc in val_locs: - splits.append('val') - elif loc in test_locs: - splits.append('test') - - print('len(img_df) = {}'.format(len(img_df))) - print('len(splits) = {}'.format(len(splits))) - - img_df = img_df.assign(split=splits) - # print(img_df.head()) - print(img_df[img_df['split']=='val']) - print(img_df[img_df['split']=='test']) - - img_df = img_df.drop_duplicates(subset=['image_id']) - - if location_coordinates is not None: - location_coordinates.columns = ['location', 'elevation', 'geometry'] - img_df = pd.merge(img_df, location_coordinates, how='left', left_on=['location'], right_on=['location']) # [location', 'date', 'image_id', 'category_id', 'filename'] - - # replace location by actual (lat, lon) coordinates - - locs = [img_df.iloc[i, -1] for i in range(len(img_df))] - locs = [np.array(x.replace('c(','').replace(')','').split(', ')).astype(float) for x in locs] - # print(locs) - - img_df.location = locs - - # print(img_df.head()) - # print(img_df.columns) - elif not args.no_location: - locs = list(img_df.location) - locs = ['{}_{}'.format(loc, args.dataset_prefix) for loc in locs] - img_df.location = locs - - - metadata = metadata.drop_duplicates(subset=['image_id'], keep=False) - - # print duplicates - # ids = metadata['image_id'] - # print(metadata[ids.isin(ids[ids.duplicated()])].sort_values('image_id')) - - print('len(img_df) = {}'.format(len(img_df))) - print('len(metadata) before = {}'.format(len(metadata))) - - metadata = pd.merge(metadata, img_df, how='inner', left_on=['image_id'], right_on=['image_id']) # [location', 'date', 'image_id', 'category_id', 'filename'] - print(metadata.columns) - print(metadata.head()) - - print('len(metadata) after = {}'.format(len(metadata))) - - # add category names - category = annotations_json['categories'] - # print('species_labels = {}'.format(species_labels)) - print('len(category) before = {}'.format(len(category))) - # print(category) - - if 'ena24' in args.data_dir: - for item in category: - item['name'] = item['name'].lower() - - category = [x for x in category if x['name'] in category_to_label_map] - - print('len(category) after = {}'.format(len(category))) - # print('category after = {}'.format([x['name'] for x in category])) - - category_ids = [x['id'] for x in category] - category_names = [taxon_name_to_id[category_to_label_map[x['name']]] for x in category] - - # for x in category_names: - # print(x in all_taxons) - # assert x in all_taxons - - category_df = pd.DataFrame(list(zip(category_ids, category_names)), columns=['category_id', 'name']) - metadata = pd.merge(metadata, category_df, how='inner', left_on=['category_id'], right_on=['category_id']) # [location', 'date', 'image_id', 'category_id', 'filename', 'name'] - - print('len(metadata) = {}'.format(len(metadata))) - - print(metadata.columns) - - if args.split_dataset: - splits = ['train'] * len(metadata) - print('len(splits) = {}'.format(len(splits))) - - n_val_samples = int(0.15 * len(metadata)) - n_test_samples = int(0.15 * len(metadata)) - splits[:n_val_samples] = ['val']*n_val_samples - splits[n_val_samples : n_val_samples+n_test_samples] = ['test']*n_test_samples - random.shuffle(splits) - - print('len(splits) = {}'.format(len(splits))) - print(splits.count('train')) - print(splits.count('val')) - print(splits.count('test')) - - metadata = metadata.assign(split=splits) - - # create category_id_to_name - category_id_to_name = {x['id']:x['name'] for x in category} - - taxon = pd.read_csv("snapshot_mountain_zebra/taxon.csv") - print('len(taxon) = {}'.format(len(taxon))) - - if not args.no_location: - takeLocation = metadata.loc[:, ['filename', 'location', 'split']] - takeLocation.insert(loc=1, column='r', value=2) - takeLocation.insert(loc=1, column='datatype_h', value='image') - takeLocation.insert(loc=4, column='datatype_t', value='location') - takeLocation.insert(loc=6, column='dataset', value=args.dataset_prefix) - takeLocation.columns = ['h', 'datatype_h', 'r', 't', 'datatype_t', 'split', 'dataset'] - print(takeLocation.head()) - - if not args.no_datetime: - takeTime = metadata.loc[:, ['filename', 'datetime', 'split']] - takeTime.insert(loc=1, column='r', value=0) - takeTime.insert(loc=1, column='datatype_h', value='image') - takeTime.insert(loc=4, column='datatype_t', value='time') - takeTime.insert(loc=6, column='dataset', value=args.dataset_prefix) - takeTime.columns = ['h', 'datatype_h', 'r', 't', 'datatype_t', 'split', 'dataset'] - print(takeTime.head()) - - imageIsIn = metadata.loc[:, ['filename', 'name', 'split']] - imageIsIn.insert(loc=1, column='r', value=3) - imageIsIn.insert(loc=1, column='datatype_h', value='image') - imageIsIn.insert(loc=4, column='datatype_t', value='id') - imageIsIn.insert(loc=6, column='dataset', value=args.dataset_prefix) - imageIsIn.columns = ['h', 'datatype_h', 'r', 't', 'datatype_t', 'split', 'dataset'] - print(imageIsIn.head()) - - dataset = pd.concat([taxon, imageIsIn, takeTime, takeLocation], ignore_index=True) - - out_file = os.path.join(args.data_dir, 'data_triples.csv') - dataset.to_csv(out_file, index=False) - diff --git a/preprocess_iwildcam.sh b/preprocess_iwildcam.sh deleted file mode 100644 index beecc88..0000000 --- a/preprocess_iwildcam.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -wget http://files.opentreeoflife.org/ott/ott3.3/ott3.3.tgz -tar -xvzf ott3.3.tgz - -python process_taxonomy_ott.py -python preprocess_data_iwildcam.py \ No newline at end of file diff --git a/preprocess_mountain_zebra.sh b/preprocess_mountain_zebra.sh deleted file mode 100644 index 073acf5..0000000 --- a/preprocess_mountain_zebra.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -wget https://www.inaturalist.org/taxa/inaturalist-taxonomy.dwca.zip -mkdir inaturalist-taxonomy.dwca/ -unzip -d inaturalist-taxonomy.dwca/ inaturalist-taxonomy.dwca.zip - -python process_taxonomy_inat.py - -mv data/snapshot_mountain_zebra/MTZ_public/MTZ_S1 data/snapshot_mountain_zebra/ - -python preprocess_data_mountain_zebra.py --data-dir data/snapshot_mountain_zebra/ --dataset-prefix mountain_zebra --species-common-names-file data/snapshot_mountain_zebra/category_to_label_map.json diff --git a/process_taxonomy_inat.py b/process_taxonomy_inat.py deleted file mode 100644 index ed299fd..0000000 --- a/process_taxonomy_inat.py +++ /dev/null @@ -1,103 +0,0 @@ -import pandas as pd -import string -from tqdm import tqdm -import re -import json - -inat_taxonomy = pd.read_csv("inaturalist-taxonomy.dwca/taxa.csv") -inat_taxonomy.fillna('') - -inat_taxonomy = inat_taxonomy.loc[:, ['id', 'parentNameUsageID', 'scientificName', 'taxonRank']] -inat_taxonomy.columns = ['uid', 'parent_uid', 'name', 'taxonRank'] - -punctuation_string = string.punctuation -taxonomy_category = list(inat_taxonomy.name) - -for i in tqdm(range(len(taxonomy_category))): - taxonomy_category[i] = ' '.join(taxonomy_category[i].split()) - taxonomy_category[i] = taxonomy_category[i].translate(str.maketrans('', '', string.punctuation)) - taxonomy_category[i] = taxonomy_category[i].lower() - taxonomy_category[i] = re.sub(' +', ' ', taxonomy_category[i]) - -inat_taxonomy_2 = inat_taxonomy.copy() -inat_taxonomy_2.name = taxonomy_category - -# replace all parent ids by just ids -parent_uids = list(inat_taxonomy_2.parent_uid) - -parent_uids_new = [] - -for x in parent_uids: - if isinstance(x, str): - parent_uids_new.append(x.replace('https://www.inaturalist.org/taxa/','')) - else: - parent_uids_new.append('') - -inat_taxonomy_2.parent_uid = parent_uids_new - -inat_taxonomy_2 = inat_taxonomy_2.loc[inat_taxonomy_2['parent_uid'] != ''] - -taxon = inat_taxonomy_2 -taxon = taxon.fillna(0) -taxon = taxon.loc[:, ["uid", "parent_uid"]] -taxon.columns = ["h", "t"] -taxon.insert(loc=1, column="r", value=1) -taxon.insert(loc=1, column="datatype_h", value="id") -taxon.insert(loc=4, column="datatype_t", value="id") -taxon.insert(loc=5, column="split", value="train") -taxon.columns = ["h", "datatype_h", "r", "t", "datatype_t", "split"] - -son = list(taxon["h"]) -father = list(taxon["t"]) -paths = {} - -for i in tqdm(range(len(son))): - if isinstance(father[i], str) and len(father[i])==0: - print('flag 1') - continue - - paths[int(float(son[i]))] = int(float(father[i])) - -taxon_id_to_name = json.load(open('data/snapshot_mountain_zebra/taxon_id_to_name_lila.json')) -category_to_label_map = json.load(open('data/snapshot_mountain_zebra/category_to_label_map_lila.json')) -taxon_name_to_id = {v:k for k,v in taxon_id_to_name.items()} - -category_names = [] - -for x in tqdm(category_to_label_map): - if category_to_label_map[x] in taxon_name_to_id: - category_names.append(taxon_name_to_id[category_to_label_map[x]]) - else: - print(category_to_label_map[x]) - -leaf_node = category_names -leaf_nodes = [] -for item in tqdm(leaf_node): - if int(float(item)) not in leaf_nodes: - leaf_nodes.append(int(float(item))) - -list_paths = [] -def get_paths(leaf_node, paths, nodes_list): - while leaf_node in paths.keys(): - # print(leaf_node,"->",paths[leaf_node]) - nodes_list.append(leaf_node) - leaf_node = paths[leaf_node] - -def get_path_nodes(leaf_nodes, paths): - nodes_list = [] - for item in leaf_nodes: - get_paths(item,paths,nodes_list) - return nodes_list - -paths_nodes = get_path_nodes(leaf_nodes, paths) - -taxon["h"] = paths.keys() -taxon["t"] = paths.values() - -taxon = taxon.loc[(taxon['h'].isin(paths_nodes)) & (taxon['t'].isin(paths_nodes)),:] -# taxon = taxon.reset_index() - -print('len(taxon) = {}'.format(len(taxon))) - -out_file = 'data/snapshot_mountain_zebra/taxon.csv' -taxon.to_csv(out_file, index=False) diff --git a/process_taxonomy_ott.py b/process_taxonomy_ott.py deleted file mode 100644 index 43a034d..0000000 --- a/process_taxonomy_ott.py +++ /dev/null @@ -1,21 +0,0 @@ -import pandas as pd -import string -from tqdm import tqdm -import re - -ott_taxonomy = pd.read_csv("ott3.3/taxonomy.tsv", sep="\t") -ott_taxonomy = ott_taxonomy.loc[:, ['uid', 'parent_uid', 'name', 'rank', 'sourceinfo', 'uniqname', 'flags']] - -punctuation_string = string.punctuation -taxonomy_category = list(ott_taxonomy.name) - -for i in tqdm(range(len(taxonomy_category))): - taxonomy_category[i] = ' '.join(taxonomy_category[i].split()) - taxonomy_category[i] = taxonomy_category[i].translate(str.maketrans('', '', string.punctuation)) - taxonomy_category[i] = taxonomy_category[i].lower() - taxonomy_category[i] = re.sub(' +', ' ', taxonomy_category[i]) - -ott_taxonomy_2 = ott_taxonomy.copy() -ott_taxonomy_2.name = taxonomy_category - -ott_taxonomy_2.to_csv('ott_taxonomy.csv', index=False) \ No newline at end of file diff --git a/pytorchtools.py b/pytorchtools.py deleted file mode 100644 index 084770e..0000000 --- a/pytorchtools.py +++ /dev/null @@ -1,84 +0,0 @@ -import numpy as np -import torch - -class EarlyStopping: - """Early stops the training if validation loss doesn't improve after a given patience.""" - def __init__(self, patience=7, verbose=False, delta=0, ckpt_path='checkpoint.pt', best_ckpt_path='best_checkpoint.pt', trace_func=print): - """ - Args: - patience (int): How long to wait after last time validation loss improved. - Default: 7 - verbose (bool): If True, prints a message for each validation loss improvement. - Default: False - delta (float): Minimum change in the monitored quantity to qualify as an improvement. - Default: 0 - path (str): Path for the checkpoint to be saved to. - Default: 'checkpoint.pt' - trace_func (function): trace print function. - Default: print - """ - self.patience = patience - self.verbose = verbose - self.counter = 0 - self.best_score = None - self.early_stop = False - self.val_loss_min = np.Inf - self.delta = delta - self.ckpt_path = ckpt_path - self.best_ckpt_path = best_ckpt_path - self.trace_func = trace_func - - def __call__(self, val_loss, model, dense_optimizer, sparse_optimizer=None): - - score = -val_loss - - if self.best_score is None: - self.best_score = score - # self.save_checkpoint(val_loss, model, dense_optimizer, sparse_optimizer) - - if self.verbose: - self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}). Saving best ckpt ...') - - if sparse_optimizer: - ckpt_dict = {'model': model.state_dict(), 'sparse_optimizer':sparse_optimizer.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - else: - ckpt_dict = {'model': model.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - - torch.save(ckpt_dict, self.best_ckpt_path) - - self.val_loss_min = val_loss - elif score < self.best_score + self.delta: - self.counter += 1 - self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}') - self.save_checkpoint(val_loss, model, dense_optimizer, sparse_optimizer) - if self.counter >= self.patience: - self.early_stop = True - else: - self.best_score = score - if self.verbose: - self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}). Saving best ckpt ...') - - if sparse_optimizer: - ckpt_dict = {'model': model.state_dict(), 'sparse_optimizer':sparse_optimizer.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - else: - ckpt_dict = {'model': model.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - - torch.save(ckpt_dict, self.best_ckpt_path) - - self.save_checkpoint(val_loss, model, dense_optimizer, sparse_optimizer) - - self.val_loss_min = val_loss - self.counter = 0 - - def save_checkpoint(self, val_loss, model, dense_optimizer, sparse_optimizer): - '''Saves model when validation loss decrease.''' - if self.verbose: - self.trace_func(f'Saving ckpt ...') - - if sparse_optimizer: - ckpt_dict = {'model': model.state_dict(), 'sparse_optimizer':sparse_optimizer.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - else: - ckpt_dict = {'model': model.state_dict(), 'dense_optimizer':dense_optimizer.state_dict()} - - torch.save(ckpt_dict, self.ckpt_path) - \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3189c94..0000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -torch==2.0.0 -pandas==1.5.3 -numpy==1.24.2 -Pillow==9.4.0 -scipy==1.10.1 -tensorboard==2.12.2 -torchvision==0.15.1 -tqdm==4.64.1 -wilds==2.0.0 -matplotlib==3.7.1 diff --git a/resnet.py b/resnet.py deleted file mode 100644 index f6280c1..0000000 --- a/resnet.py +++ /dev/null @@ -1,33 +0,0 @@ -import torch -import torch.nn as nn -from torchvision.models import resnet18, resnet50 - -class Resnet18(nn.Module): - def __init__(self, args): - super(Resnet18, self).__init__() - self.args = args - self.image_embedding = resnet18(pretrained=True) - self.needs_y = False - - self.image_embedding.fc = nn.Linear(512, 182) - nn.init.xavier_uniform_(self.image_embedding.fc.weight.data) - - def forward(self, x): - emb_h = self.image_embedding(x) - - return emb_h - -class Resnet50(nn.Module): - def __init__(self, args): - super(Resnet50, self).__init__() - self.args = args - self.image_embedding = resnet50(pretrained=True) - self.needs_y = False - - self.image_embedding.fc = nn.Linear(2048, 182) - nn.init.xavier_uniform_(self.image_embedding.fc.weight.data) - - def forward(self, x): - emb_h = self.image_embedding(x) - - return emb_h \ No newline at end of file diff --git a/run_image_only_model.py b/run_image_only_model.py deleted file mode 100644 index 6f8c87b..0000000 --- a/run_image_only_model.py +++ /dev/null @@ -1,269 +0,0 @@ -import os -import time -import argparse -import numpy as np -import random -import pandas as pd -import torch -import torch.nn as nn -import torchvision -import sys -from collections import defaultdict -import math -import torchvision.transforms as transforms - -from resnet import Resnet50 - -from tqdm import tqdm -from utils import collate_list, detach_and_clone, move_to -import torch.optim as optim -from torch.utils.data import Dataset, DataLoader -from wilds.common.metrics.all_metrics import Accuracy -from PIL import Image -from pytorchtools import EarlyStopping - -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN = [0.485, 0.456, 0.406] -_DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD = [0.229, 0.224, 0.225] - -def run_epoch(model, train_loader, val_loader, optimizer, epoch, args, early_stopping, train): - - if train: - model.train() - torch.set_grad_enabled(True) - else: - model.eval() - torch.set_grad_enabled(False) - - epoch_y_true = [] - epoch_y_pred = [] - - batches = train_loader if train else val_loader - - batches = tqdm(batches) - last_batch_idx = len(batches)-1 - - criterion = nn.CrossEntropyLoss() - - batch_idx = 0 - for labeled_batch in batches: - if train: - x, y_true = labeled_batch - x = move_to(x, args.device) - y_true = move_to(y_true, args.device) - - outputs = model(x) - - batch_results = { - # 'g': g, - 'y_true': y_true.cpu(), - 'y_pred': outputs.cpu(), - # 'metadata': metadata, - } - - # compute objective - loss = criterion(batch_results['y_pred'], batch_results['y_true']) - batch_results['objective'] = loss.item() - loss.backward() - - # update model and logs based on effective batch - optimizer.step() - model.zero_grad() - - else: - x, y_true = labeled_batch - x = move_to(x, args.device) - y_true = move_to(y_true, args.device) - - outputs = model(x) - - batch_results = { - # 'g': g, - 'y_true': y_true.cpu(), - 'y_pred': outputs.cpu(), - # 'metadata': metadata, - } - - batch_results['objective'] = criterion(batch_results['y_pred'], batch_results['y_true']).item() - - epoch_y_true.append(detach_and_clone(batch_results['y_true'])) - y_pred = detach_and_clone(batch_results['y_pred']) - y_pred = y_pred.argmax(-1) - - epoch_y_pred.append(y_pred) - - effective_batch_idx = batch_idx + 1 - - batch_idx += 1 - if args.debug and batch_idx > 100: - break - - epoch_y_pred = collate_list(epoch_y_pred) - epoch_y_true = collate_list(epoch_y_true) - # epoch_metadata = collate_list(epoch_metadata) - - metrics = [ - Accuracy(prediction_fn=None), - ] - - results = {} - - for i in range(len(metrics)): - results.update({ - **metrics[i].compute(epoch_y_pred, epoch_y_true), - }) - - results_str = ( - f"Average acc: {results[metrics[0].agg_metric_field]:.3f}\n" - ) - - if not train: # just for eval. - early_stopping(-1*results[metrics[0].agg_metric_field], model, optimizer) - - results['epoch'] = epoch - # if dataset['verbose']: - print('Epoch eval:\n') - print(results_str) - - return results, epoch_y_pred - -class iWildCamDataset(Dataset): - def __init__(self, datacsv, img_dir, mode, entity2id, target_list): # dic_data <- datas - super(iWildCamDataset, self).__init__() - self.mode = mode - self.datacsv = datacsv.loc[datacsv['split'] == mode, :] - self.img_dir = img_dir - self.entity2id = entity2id - self.target_list = target_list - self.entity_to_species_id = {self.target_list[i, 0].item():i for i in range(len(self.target_list))} - - def __len__(self): - return len(self.datacsv) - - def __getitem__(self, idx): - y = torch.tensor([self.entity_to_species_id[self.entity2id[str(int(float(self.datacsv.iloc[idx, 3])))]]], dtype=torch.long).squeeze() - - img = Image.open(os.path.join(self.img_dir, self.datacsv.iloc[idx, 0])).convert('RGB') - - transform_steps = transforms.Compose([transforms.Resize((448, 448)), transforms.ToTensor(), transforms.Normalize(_DEFAULT_IMAGE_TENSOR_NORMALIZATION_MEAN, _DEFAULT_IMAGE_TENSOR_NORMALIZATION_STD)]) - x = transform_steps(img) - - return x, y - -def _get_id(dict, key): - id = dict.get(key, None) - if id is None: - id = len(dict) - dict[key] = id - return id - -def generate_target_list(data, entity2id): - sub = data.loc[(data["datatype_h"] == "image") & (data["datatype_t"] == "id"), ['t']] - sub = list(sub['t']) - categories = [] - for item in tqdm(sub): - if entity2id[str(int(float(item)))] not in categories: - categories.append(entity2id[str(int(float(item)))]) - print("No. of target categories = {}".format(len(categories))) - return torch.tensor(categories, dtype=torch.long).unsqueeze(-1) - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument('--dataset', choices=['iwildcam', 'mountain_zebra'], default='iwildcam') - parser.add_argument('--data-dir', type=str, default='iwildcam_v2.0/') - parser.add_argument('--img-dir', type=str, default='iwildcam_v2.0/imgs/') - parser.add_argument('--batch_size', type=int, default=16) - parser.add_argument('--n_epochs', type=int, default=12) - parser.add_argument('--lr', type=float, default=3e-5) - parser.add_argument('--weight_decay', type=float, default=0.0) - parser.add_argument('--device', type=int, nargs='+', default=[0]) - parser.add_argument('--seed', type=int, default=813765) - parser.add_argument('--save-dir', type=str, default='ckpts/toy/') - parser.add_argument('--debug', action='store_true') - parser.add_argument('--early-stopping-patience', type=int, default=5, help='early stop if metric does not improve for x epochs') - - args = parser.parse_args() - - print('args = {}'.format(args)) - - # Set device - if torch.cuda.is_available(): - device_count = torch.cuda.device_count() - if len(args.device) > device_count: - raise ValueError(f"Specified {len(args.device)} devices, but only {device_count} devices found.") - - device_str = ",".join(map(str, args.device)) - os.environ["CUDA_VISIBLE_DEVICES"] = device_str - args.device = torch.device("cuda") - else: - args.device = torch.device("cpu") - - # Set random seed - # set_seed(args.seed) - torch.manual_seed(args.seed) - np.random.seed(args.seed) - random.seed(args.seed) - - if args.dataset == 'iwildcam': - datacsv = pd.read_csv(os.path.join(args.data_dir, 'dataset_subtree.csv'), low_memory=False) - else: - datacsv = pd.read_csv(os.path.join(args.data_dir, 'data_triples.csv'), low_memory=False) - - datacsv = datacsv.loc[(datacsv["datatype_h"] == "image") & (datacsv["datatype_t"] == "id")] - - entity2id = {} # each of triple types have their own entity2id - - for i in tqdm(range(datacsv.shape[0])): - _get_id(entity2id, str(int(float(datacsv.iloc[i,3])))) - - print('len(entity2id) = {}'.format(len(entity2id))) - - target_list = generate_target_list(datacsv, entity2id) - - train_dataset = iWildCamDataset(datacsv, args.img_dir, 'train', entity2id, target_list) - val_dataset = iWildCamDataset(datacsv, args.img_dir, 'val', entity2id, target_list) - - - train_loader = DataLoader( - train_dataset, - shuffle=True, # Shuffle training dataset - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True) - - val_loader = DataLoader( - val_dataset, - shuffle=False, # Do not shuffle eval datasets - sampler=None, - batch_size=args.batch_size, - num_workers=4, - pin_memory=True) - - model = Resnet50(args) - model.to(args.device) - - params = filter(lambda p: p.requires_grad, model.parameters()) - optimizer = optim.Adam( - params, - lr=args.lr, - weight_decay=args.weight_decay) - - best_val_metric = None - early_stopping = EarlyStopping(patience=args.early_stopping_patience, verbose=True, ckpt_path=os.path.join(args.save_dir, 'model.pt'), best_ckpt_path=os.path.join(args.save_dir, 'best_model.pt')) - - for epoch in range(args.n_epochs): - print('\nEpoch [%d]:\n' % epoch) - - # First run training - run_epoch(model, train_loader, val_loader, optimizer, epoch, args, early_stopping, train=True) - - # Then run val - val_results, y_pred = run_epoch(model, train_loader, val_loader, optimizer, epoch, args, early_stopping, train=False) - - if early_stopping.early_stop: - print("Early stopping...") - break - -if __name__=='__main__': - main() diff --git a/utils.py b/utils.py deleted file mode 100644 index 2dc275a..0000000 --- a/utils.py +++ /dev/null @@ -1,74 +0,0 @@ -import sys -import os -import csv -import argparse -import random -from pathlib import Path -import numpy as np -import torch -import pandas as pd -import re - -from torch.utils.data import DataLoader - -try: - from torch_geometric.data import Batch -except ImportError: - pass - -def set_seed(seed): - """Sets seed""" - if torch.cuda.is_available(): - torch.cuda.manual_seed(seed) - torch.manual_seed(seed) - np.random.seed(seed) - random.seed(seed) - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - -def move_to(obj, device): - if isinstance(obj, dict): - return {k: move_to(v, device) for k, v in obj.items()} - elif isinstance(obj, list): - return [move_to(v, device) for v in obj] - elif isinstance(obj, float) or isinstance(obj, int): - return obj - else: - # Assume obj is a Tensor or other type - # (like Batch, for MolPCBA) that supports .to(device) - return obj.to(device) - -def detach_and_clone(obj): - if torch.is_tensor(obj): - return obj.detach().clone() - elif isinstance(obj, dict): - return {k: detach_and_clone(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [detach_and_clone(v) for v in obj] - elif isinstance(obj, float) or isinstance(obj, int): - return obj - else: - raise TypeError("Invalid type for detach_and_clone") - -def collate_list(vec): - """ - If vec is a list of Tensors, it concatenates them all along the first dimension. - - If vec is a list of lists, it joins these lists together, but does not attempt to - recursively collate. This allows each element of the list to be, e.g., its own dict. - - If vec is a list of dicts (with the same keys in each dict), it returns a single dict - with the same keys. For each key, it recursively collates all entries in the list. - """ - if not isinstance(vec, list): - raise TypeError("collate_list must take in a list") - elem = vec[0] - if torch.is_tensor(elem): - return torch.cat(vec) - elif isinstance(elem, list): - return [obj for sublist in vec for obj in sublist] - elif isinstance(elem, dict): - return {k: collate_list([d[k] for d in vec]) for k in elem} - else: - raise TypeError("Elements of the list to collate must be tensors or dicts.")