Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make vocabularies multilingual #600

Merged
merged 12 commits into from
Aug 8, 2022
Merged

Make vocabularies multilingual #600

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8cbfffa
get available languages from SKOS vocabulary
osma Aug 3, 2022
72dd66b
make SubjectFileSKOS language-agnostic
osma Aug 3, 2022
f38ac54
make subject index filenames language-specific, e.g. subjects.en.tsv
osma Aug 3, 2022
a0ea2d3
store subject index in all available (SKOS) vocab languages
osma Aug 4, 2022
b67bb1c
use language-agnostic vocabulary id's in test configuration
osma Aug 4, 2022
cca7ca8
enable overriding of vocab language in configuration, e.g. lcsh(en)
osma Aug 4, 2022
4d78bb5
handle case where SKOS lacks language tags & fix comments
osma Aug 4, 2022
7388f78
refactor languages property
osma Aug 5, 2022
8204368
refactor vocabulary creation
osma Aug 5, 2022
baa3b2f
refactor languages property (again), now using set comprehension
osma Aug 5, 2022
4e62fc9
refactor load_vocabulary slightly
osma Aug 5, 2022
b258873
Add test for invalid vocabulary specifier
osma Aug 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor vocabulary creation
  • Loading branch information
@osma
osma committed Aug 5, 2022
commit 820436873020dfb1c70660a9baa92898040404ef
15 changes: 3 additions & 12 deletions annif/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import enum
import os.path
import re
from shutil import rmtree
import annif
import annif.transform
Expand All @@ -14,7 +13,6 @@
from annif.datadir import DatadirMixin
from annif.exception import AnnifException, ConfigurationException, \
NotSupportedException, NotInitializedException
from annif.util import parse_args

logger = annif.logger

Expand Down Expand Up @@ -157,17 +155,10 @@ def vocab(self):
if self.vocab_spec is None:
raise ConfigurationException("vocab setting is missing",
project_id=self.project_id)
match = re.match(r'(\w+)(\((.*)\))?', self.vocab_spec)
if match is None:
raise ConfigurationException("vocab setting is invalid",
project_id=self.project_id)
vocab_id = match.group(1)
posargs, kwargs = parse_args(match.group(3))
language = posargs[0] if posargs else self.language
self._vocab = annif.vocab.get_vocab(self.vocab_spec,
self._base_datadir,
self.language)

self._vocab = annif.vocab.AnnifVocabulary(vocab_id,
self._base_datadir,
language)
return self._vocab

@property
Expand Down
13 changes: 13 additions & 0 deletions annif/vocab.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,28 @@
"""Vocabulary management functionality for Annif"""

import os.path
import re
import annif
import annif.corpus
import annif.util
from annif.datadir import DatadirMixin
from annif.exception import NotInitializedException
from annif.util import parse_args

logger = annif.logger


def get_vocab(vocab_spec, datadir, default_language):
match = re.match(r'(\w+)(\((.*)\))?', vocab_spec)
if match is None:
raise ValueError(f"Invalid vocabulary specification: {vocab_spec}")
vocab_id = match.group(1)
posargs, kwargs = parse_args(match.group(3))
language = posargs[0] if posargs else default_language

return AnnifVocabulary(vocab_id, datadir, language)


class AnnifVocabulary(DatadirMixin):
"""Class representing a subject vocabulary which can be used by multiple
Annif projects."""
Expand Down