Skip to content

Commit

Permalink
Automatically download models on demand
Browse files Browse the repository at this point in the history
  • Loading branch information
movabo committed Dec 8, 2021
1 parent 181b970 commit ea6766f
Show file tree
Hide file tree
Showing 110 changed files with 209 additions and 207 deletions.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
import jsonlines
from tabulate import tabulate

from DeepNewsSentiment.SentimentClasses import SentimentClasses
from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.fxlogger import get_logger


class DatasetPreparer:
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
4 changes: 2 additions & 2 deletions DeepNewsSentiment/__main__.py → NewsSentiment/__main__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import argparse

from DeepNewsSentiment.download import Download
from NewsSentiment.download import Download

if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='DeepNewsAnalysis')
parser = argparse.ArgumentParser(prog='NewsSentiment')
subparsers = parser.add_subparsers(dest='action')

subparser_download = subparsers.add_parser('download', help=Download.add_subparser.__doc__)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from DeepNewsSentiment.consts import BERT_BASE_UNCASED
from NewsSentiment.consts import BERT_BASE_UNCASED

combinations_default_0 = {
"own_model_name": [
Expand Down
File renamed without changes.
36 changes: 18 additions & 18 deletions DeepNewsSentiment/controller.py → NewsSentiment/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,24 @@
from tabulate import tabulate
from tqdm import tqdm

from DeepNewsSentiment.combinations_ensemble import combinations_ensemble
from DeepNewsSentiment.combinations_final_lcf2 import combinations_final_lcf2
from DeepNewsSentiment.combinations_final_nostance import combinations_final_nostance
from DeepNewsSentiment.combinations_final_spc import combinations_final_spc
from DeepNewsSentiment.combinations_final_stance0 import combinations_final_stance0
from DeepNewsSentiment.combinations_final_stance1 import combinations_final_stance1
from DeepNewsSentiment.combinations_final_tdbert import combinations_final_tdbert
from DeepNewsSentiment.combinations_top import combinations_top
from DeepNewsSentiment.DatasetPreparer import DatasetPreparer
from DeepNewsSentiment.combinations_default import combinations_default_0
from DeepNewsSentiment.combinations_top4otherdomains import combinations_top4otherdomains
from DeepNewsSentiment.combinations_top_coref16 import combinations_top_coref16
from DeepNewsSentiment.combinations_top_coref8 import combinations_top_coref8
from DeepNewsSentiment.combinations_top_coref8_stanceno import combinations_top_coref8_stanceno
from DeepNewsSentiment.combinations_top_finetuned import combinations_top_finetuned
from DeepNewsSentiment.diskdict import DiskDict
from DeepNewsSentiment.fxlogger import get_logger
from DeepNewsSentiment.train import str2bool
from NewsSentiment.combinations_ensemble import combinations_ensemble
from NewsSentiment.combinations_final_lcf2 import combinations_final_lcf2
from NewsSentiment.combinations_final_nostance import combinations_final_nostance
from NewsSentiment.combinations_final_spc import combinations_final_spc
from NewsSentiment.combinations_final_stance0 import combinations_final_stance0
from NewsSentiment.combinations_final_stance1 import combinations_final_stance1
from NewsSentiment.combinations_final_tdbert import combinations_final_tdbert
from NewsSentiment.combinations_top import combinations_top
from NewsSentiment.DatasetPreparer import DatasetPreparer
from NewsSentiment.combinations_default import combinations_default_0
from NewsSentiment.combinations_top4otherdomains import combinations_top4otherdomains
from NewsSentiment.combinations_top_coref16 import combinations_top_coref16
from NewsSentiment.combinations_top_coref8 import combinations_top_coref8
from NewsSentiment.combinations_top_coref8_stanceno import combinations_top_coref8_stanceno
from NewsSentiment.combinations_top_finetuned import combinations_top_finetuned
from NewsSentiment.diskdict import DiskDict
from NewsSentiment.fxlogger import get_logger
from NewsSentiment.train import str2bool

completed_tasks = None # will be DiskDict later
completed_tasks_in_this_run_count = 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import jsonlines
import pandas as pd

from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.fxlogger import get_logger

logger = get_logger()

Expand Down
File renamed without changes.
18 changes: 9 additions & 9 deletions DeepNewsSentiment/dataset.py → NewsSentiment/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,26 @@
AlbertTokenizer,
)

from DeepNewsSentiment.SentimentClasses import SentimentClasses
from DeepNewsSentiment.customexceptions import TooLongTextException, TargetNotFoundException
from DeepNewsSentiment.consts import *
from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.customexceptions import TooLongTextException, TargetNotFoundException
from NewsSentiment.consts import *
from NewsSentiment.fxlogger import get_logger

# get logger
from DeepNewsSentiment.knowledge.bingliuopinion.bingliuopinion import (
from NewsSentiment.knowledge.bingliuopinion.bingliuopinion import (
get_bingliu_polarities_as_tensor,
get_num_bingliu_polarities,
)
from DeepNewsSentiment.knowledge.mpqasubjectivity.mpqasubjectivity import (
from NewsSentiment.knowledge.mpqasubjectivity.mpqasubjectivity import (
get_mpqa_subjectivity_polarities_as_tensor,
get_num_mpqa_subjectivity_polarities,
)
from DeepNewsSentiment.knowledge.nrcemolex.nrcemolex import (
from NewsSentiment.knowledge.nrcemolex.nrcemolex import (
get_nrc_emotions_as_tensor,
get_num_nrc_emotions,
)
from DeepNewsSentiment.knowledge.zeros.zerosknowledge import get_num_zero_dimensions, get_zeros_as_tensor
from DeepNewsSentiment.models.FXBaseModel import FXBaseModel
from NewsSentiment.knowledge.zeros.zerosknowledge import get_num_zero_dimensions, get_zeros_as_tensor
from NewsSentiment.models.FXBaseModel import FXBaseModel

logger = get_logger()
try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import pickle
from typing import Union

from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.fxlogger import get_logger

logger = get_logger()

Expand Down
13 changes: 8 additions & 5 deletions DeepNewsSentiment/download.py → NewsSentiment/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@

import torch

from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.fxlogger import get_logger


class Download:

def __init__(
self, own_model_name, version="default", force=False, list_versions=False
):
from DeepNewsSentiment.models.FXBaseModel import model_includes_pretrained
from DeepNewsSentiment.train import OWN_MODELNAME2CLASS
from NewsSentiment.models.FXBaseModel import model_includes_pretrained
from NewsSentiment.train import OWN_MODELNAME2CLASS

logger = get_logger()
own_model_name = own_model_name
Expand Down Expand Up @@ -50,15 +50,18 @@ def list_versions(model_cls, own_model_name=""):
print(f'"{version}"{default_str}: {source}')

@classmethod
def download(cls, model_cls, version=None, force=False):
def download(cls, model_cls, version=None, force=False, skip_if_exists=True):
source = model_cls.get_pretrained_source(version)
path = cls.model_path(model_cls, version)
if not force and os.path.isfile(path):
if skip_if_exists:
return
print("Model file already exists. Use --force to overwrite.")
exit(2)
print(f"Downloading to {path}:")
os.makedirs(os.path.dirname(path), exist_ok=True)
torch.hub.download_url_to_file(source, path)
return path

@staticmethod
def model_filename(model_cls, version=None):
Expand All @@ -78,7 +81,7 @@ def model_path(cls, model_cls, version=None):

@staticmethod
def add_subparser(subparser):
"""Download models for DeepNewsSentiment"""
"""Download models for NewsSentiment"""
subparser.add_argument("--own_model_name", default="grutsc", type=str)
subparser.add_argument(
"--version",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Copyright by Bjarten
# License: MIT

from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.fxlogger import get_logger


class EarlyStopping:
Expand Down
6 changes: 3 additions & 3 deletions DeepNewsSentiment/evaluator.py → NewsSentiment/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
import numpy as np
from sklearn import metrics

from DeepNewsSentiment.SentimentClasses import SentimentClasses
from DeepNewsSentiment.dataset import FXDataset
from DeepNewsSentiment.fxlogger import get_logger
from NewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.dataset import FXDataset
from NewsSentiment.fxlogger import get_logger


class Evaluator:
Expand Down
4 changes: 2 additions & 2 deletions DeepNewsSentiment/fxlogger.py → NewsSentiment/fxlogger.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ def get_logger():
handler = logging.StreamHandler()
handler.setFormatter(formatter)

logger = logging.getLogger("DeepNewsSentiment")
logger = logging.getLogger("NewsSentiment")
if logger.level == logging.NOTSET:
logger.setLevel(logging.INFO)
logger.setLevel(logging.ERROR)

if len(logger.handlers) == 0:
logger.addHandler(handler)
Expand Down
14 changes: 7 additions & 7 deletions DeepNewsSentiment/infer.py → NewsSentiment/infer.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
from jsonlines import jsonlines
from tqdm import tqdm

from DeepNewsSentiment.SentimentClasses import SentimentClasses
from DeepNewsSentiment.dataset import FXEasyTokenizer
from DeepNewsSentiment.download import Download
from DeepNewsSentiment.fxlogger import get_logger
from DeepNewsSentiment.models.singletarget.grutscsingle import GRUTSCSingle
from DeepNewsSentiment.train import (
from NewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.dataset import FXEasyTokenizer
from NewsSentiment.download import Download
from NewsSentiment.fxlogger import get_logger
from NewsSentiment.models.singletarget.grutscsingle import GRUTSCSingle
from NewsSentiment.train import (
parse_arguments as parse_arguments_from_train,
prepare_and_start_instructor,
)
Expand Down Expand Up @@ -142,7 +142,7 @@ def parse_arguments(override_args=False):
)
parser.add_argument("--default_lm", default="roberta-base", type=str)
parser.add_argument(
"--state_dict", type=str, default=Download.model_filename(GRUTSCSingle)
"--state_dict", type=str, default=Download.model_path(GRUTSCSingle)
)
parser.add_argument(
"--knowledgesources", default="nrc_emotions mpqa_subjectivity bingliu_opinion"
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import torch

from DeepNewsSentiment.diskdict import DiskDict
from DeepNewsSentiment.knowledge.bingliuopinion.converter import PATH_DICT_BING_LIU_OPINION_POLARITY, POLARITY2INDEX
from DeepNewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc
from NewsSentiment.diskdict import DiskDict
from NewsSentiment.knowledge.bingliuopinion.converter import PATH_DICT_BING_LIU_OPINION_POLARITY, POLARITY2INDEX
from NewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc

__ddict_emolex = DiskDict(PATH_DICT_BING_LIU_OPINION_POLARITY)
__ddict_emolex_keys_lower = {k.lower(): v for k, v in __ddict_emolex.items()}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@

from tqdm import tqdm

from DeepNewsSentiment.fxlogger import get_logger
from DeepNewsSentiment.diskdict import DiskDict
from NewsSentiment.fxlogger import get_logger
from NewsSentiment.diskdict import DiskDict

POLARITY2INDEX = {
"positive": 1,
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import torch

from DeepNewsSentiment.knowledge.liwc.liwchelper import load_token_parser
from NewsSentiment.knowledge.liwc.liwchelper import load_token_parser

parse, category_names = load_token_parser()

Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@

from tqdm import tqdm

from DeepNewsSentiment.fxlogger import get_logger
from DeepNewsSentiment.diskdict import DiskDict
from NewsSentiment.fxlogger import get_logger
from NewsSentiment.diskdict import DiskDict

POLARITY2INDEX = {
"positive": 2,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import torch

from DeepNewsSentiment.diskdict import DiskDict
from DeepNewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc
from DeepNewsSentiment.knowledge.mpqasubjectivity.converter import POLARITY2INDEX, PATH_DICT_MPQA_SUBJECTIVITY
from NewsSentiment.diskdict import DiskDict
from NewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc
from NewsSentiment.knowledge.mpqasubjectivity.converter import POLARITY2INDEX, PATH_DICT_MPQA_SUBJECTIVITY

__ddict_emolex = DiskDict(PATH_DICT_MPQA_SUBJECTIVITY)
__ddict_emolex_keys_lower = {k.lower(): v for k, v in __ddict_emolex.items()}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
from pathlib import Path
import csv

from DeepNewsSentiment.fxlogger import get_logger
from DeepNewsSentiment.diskdict import DiskDict
from NewsSentiment.fxlogger import get_logger
from NewsSentiment.diskdict import DiskDict

EMOTION2INDEX = {
"anger": 0,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import torch

from DeepNewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc
from DeepNewsSentiment.knowledge.nrcemolex.converter import PATH_DICT_NRC_EMOLEX, EMOTION2INDEX
from DeepNewsSentiment.diskdict import DiskDict
from NewsSentiment.knowledge.knowledgeutils import find_key_original_or_lc
from NewsSentiment.knowledge.nrcemolex.converter import PATH_DICT_NRC_EMOLEX, EMOTION2INDEX
from NewsSentiment.diskdict import DiskDict

__ddict_emolex = DiskDict(PATH_DICT_NRC_EMOLEX)
__ddict_emolex_keys_lower = {k.lower(): v for k, v in __ddict_emolex.items()}
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import torch.nn as nn

# CrossEntropyLoss with additional cross weight loss for two targets
from DeepNewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.SentimentClasses import SentimentClasses


class CrossEntropyLoss_CrossWeight(nn.Module):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import torch.nn as nn

# CrossEntropyLoss for Label Smoothing Regularization
from DeepNewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.SentimentClasses import SentimentClasses


class CrossEntropyLoss_LSR(nn.Module):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import torch
import torch.nn as nn

from DeepNewsSentiment.SentimentClasses import SentimentClasses
from NewsSentiment.SentimentClasses import SentimentClasses


class SequenceLoss(nn.Module):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from transformers import XLNetModel, AlbertModel, BertModel, RobertaModel
from functools import wraps

from DeepNewsSentiment.download import Download
from NewsSentiment.download import Download


class FXBaseModel(nn.Module):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
import torch.nn as nn
import torch

from DeepNewsSentiment.consts import *
from DeepNewsSentiment.dataset import FXDataset
from DeepNewsSentiment.models.FXBaseModel import FXBaseModel
from NewsSentiment.consts import *
from NewsSentiment.dataset import FXDataset
from NewsSentiment.models.FXBaseModel import FXBaseModel


class FXEnsemble(FXBaseModel):
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from argparse import Namespace
from typing import List, Dict
from DeepNewsSentiment.consts import *
from NewsSentiment.consts import *
import torch.nn as nn
import torch

from DeepNewsSentiment.models.FXBaseModel import FXBaseModel
from DeepNewsSentiment.models.singletarget.td_bert import TD_BERT
from NewsSentiment.models.FXBaseModel import FXBaseModel
from NewsSentiment.models.singletarget.td_bert import TD_BERT


class EnsembleTopA(FXBaseModel):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from argparse import Namespace
from typing import List, Dict
from DeepNewsSentiment.consts import *
from NewsSentiment.consts import *
import torch.nn as nn
import torch

from DeepNewsSentiment.models.FXBaseModel import FXBaseModel
from DeepNewsSentiment.models.singletarget.td_bert import TD_BERT
from NewsSentiment.models.FXBaseModel import FXBaseModel
from NewsSentiment.models.singletarget.td_bert import TD_BERT


class EnsembleTopB(FXBaseModel):
Expand Down
Loading

0 comments on commit ea6766f

Please sign in to comment.