r3m and imagenet done

notvenky · Mar 13, 2024 · 6fa5d09 · 6fa5d09
1 parent 623af36
commit 6fa5d09
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 3 deletions.
diff --git a/baselines/imagenet.py b/baselines/imagenet.py
@@ -0,0 +1,39 @@
+import torch
+import torchvision.transforms as T
+import torchvision.models as models
+from PIL import Image
+import h5py
+import os
+import datetime
+
+if torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+model = models.resnet50(pretrained=True)  # Ensure pretrained=True to load the ImageNet trained weights
+model.eval()
+model.to(device)
+
+transforms = T.Compose([T.Resize(256),
+    T.CenterCrop(224),
+    T.ToTensor()])
+
+image_path = '/home/venky/CogVLM/test_set/d5.jpg'
+image = Image.open(image_path).convert('RGB')
+preprocessed_image = transforms(image).reshape(-1, 3, 224, 224)
+preprocessed_image = preprocessed_image.to(device)
+
+with torch.no_grad():
+    embedding = model(preprocessed_image * 255.0)
+
+print(embedding.shape)
+
+# Saving the embedding, similar to the original script
+img_name = os.path.splitext(os.path.basename(image_path))[0]
+current_time = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+run_name = f"imagenet_{img_name}_{current_time}"
+run_dir_path = os.path.join('/home/venky/CogVLM/baselines/logs', run_name)
+os.makedirs(run_dir_path, exist_ok=True)
+with h5py.File(os.path.join(run_dir_path, 'embeddings.h5'), 'w') as f:
+    f.create_dataset('embeddings', data=embedding.cpu().numpy())
diff --git a/baselines/model_mvp.py b/baselines/model_mvp.py
@@ -1,6 +1,35 @@
+import torch
 import mvp
 
-model = mvp.load("vitb-mae-egosoup")
+import os
+import h5py
+import numpy as np
+from PIL import Image
+import datetime
+import torchvision.transforms as T
+import torchvision.models as models
+
+model = mvp.
 model.freeze()
+model.to("cuda")
+
+transforms = T.Compose([T.Resize(256),
+    T.CenterCrop(224),
+    T.ToTensor()])
+
+image_path = '/home/venky/CogVLM/test_set/d1.jpg'
+image = Image.open(image_path).convert('RGB')
+preprocessed_image = transforms(image).reshape(-1, 3, 224, 224)
+preprocessed_image = preprocessed_image.to("cuda")
+
+with torch.no_grad():
+    embedding = model(preprocessed_image)
+print(embedding.shape)
 
-import ipdb; ipdb.set_trace()
+img_name = os.path.splitext(os.path.basename(image_path))[0]
+current_time = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+run_name = f"mvp_{img_name}_{current_time}"
+run_dir_path = os.path.join('/home/venky/CogVLM/baselines/logs', run_name)
+os.makedirs(run_dir_path, exist_ok=True)
+with h5py.File(os.path.join(run_dir_path, 'embeddings.h5'), 'w') as f:
+    f.create_dataset('embeddings', data=embedding.cpu().numpy())
diff --git a/baselines/mvpmode.py b/baselines/mvpmode.py
@@ -0,0 +1,5 @@
+import mvp
+
+model = mvp.load("vitb-mae-egosoup")
+model.freeze()
+import ipdb; ipdb.set_trace()
diff --git a/cossim.py b/cossim.py
@@ -48,12 +48,20 @@ def compute_cosine_sim_for_hidden_states(tensor_file_1, tensor_file_2, key):
 
     return cosine_sim.mean()
 
+def cosine_sim_embeddings(tensor_file_1, tensor_file_2):
+    tensor_1 = load_tensor_from_hdf5(tensor_file_1, 'embeddings')
+    tensor_2 = load_tensor_from_hdf5(tensor_file_2, 'embeddings')
+
+    cosine_sim = torch.nn.functional.cosine_similarity(tensor_1, tensor_2, dim=1)
+    return cosine_sim
+
 if __name__ == '__main__':
     # keys = ['cross_attn_weights', 'self_attn_weights']
     keys = ['hidden_states']
     # keys = ['output_embeddings']
     for _ in keys:
         # similarity = compute_cosine_sim_for_key('/home/venky/CogVLM/logs/2024-03-07/run_d4_dist_2024-03-07_10-40-55/d4_dist_intermediate_representations.h5', '/home/venky/CogVLM/logs/2024-03-07/run_d1_2024-03-07_10-37-04/d1_intermediate_representations.h5', _)
-        similarity = compute_cosine_sim_for_hidden_states('/home/venky/CogVLM/logs/2024-03-13/N8_p3_d5_2024-03-13_14-50-05/d5_intermediate_representations.h5', '/home/venky/CogVLM/logs/2024-03-13/N8_p3_d1_2024-03-13_14-48-56/d1_intermediate_representations.h5', _)
+        # similarity = compute_cosine_sim_for_hidden_states('/home/venky/CogVLM/logs/2024-03-13/N8_p3_d5_2024-03-13_14-50-05/d5_intermediate_representations.h5', '/home/venky/CogVLM/logs/2024-03-13/N8_p3_d1_2024-03-13_14-48-56/d1_intermediate_representations.h5', _)
+        similarity = cosine_sim_embeddings('/home/venky/CogVLM/baselines/logs/imagenet_d3_2024-03-13_16-09-26/embeddings.h5', '/home/venky/CogVLM/baselines/logs/imagenet_d1_2024-03-13_16-08-59/embeddings.h5')
         print(f"Cosine similarity for {_}:", similarity)
     print("Overall Similarity Score:", similarity)